<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import logging
import re

import os
import requests
from builtins import str
from typing import Text, List, Dict, Any

from rasa_core import constants
from rasa_core.utils import EndpointConfig

logger = logging.getLogger(__name__)

INTENT_MESSAGE_PREFIX = "/"


class NaturalLanguageInterpreter(object):
    def parse(self, text):
        raise NotImplementedError(
                "Interpreter needs to be able to parse "
                "messages into structured output.")

    @staticmethod
    def create(obj):
        if isinstance(obj, NaturalLanguageInterpreter):
            return obj
        if isinstance(obj, str):
            return RasaNLUInterpreter(model_directory=obj)
        else:
            return RegexInterpreter()  # default interpreter


class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX + "_"   # _ is deprecated but supported

    @staticmethod
    def _create_entities(parsed_entities, sidx, eidx):
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append({
                    "entity": k,
                    "start": sidx,
                    "end": eidx,  # can't be more specific
                    "value": value
                })
        return entities

    @staticmethod
    def _parse_parameters(entitiy_str, sidx, eidx, user_input):
        # type: (Text, int, int, Text) -> List[Dict[Text, Any]]
        if entitiy_str is None or not entitiy_str.strip():
            # if there is nothing to parse we will directly exit
            return []

        try:
            parsed_entities = json.loads(entitiy_str)
            if isinstance(parsed_entities, dict):
                return RegexInterpreter._create_entities(parsed_entities,
                                                         sidx, eidx)
            else:
                raise Exception("Parsed value isn't a json object "
                                "(instead parser found '{}')"
                                ".".format(type(parsed_entities)))
        except Exception as e:
            logger.warning("Invalid to parse arguments in line "
                           "'{}'. Failed to decode parameters"
                           "as a json object. Make sure the intent"
                           "followed by a proper json object. "
                           "Error: {}".format(user_input, e))
            return []

    @staticmethod
    def _parse_confidence(confidence_str):
        # type: (Text) -> float
        if confidence_str is None:
            return 1.0
        return float(confidence_str.strip()[1:])

    @staticmethod
    def extract_intent_and_entities(user_input):
        # type: (Text) -> object
        """Parse the user input using regexes to extract intent & entities."""

        prefixes = re.escape(RegexInterpreter.allowed_prefixes())<commit_msg>Handle exceptions in intent confidence parsing<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import logging
import re

import os
import requests
from builtins import str
from typing import Text, List, Dict, Any

from rasa_core import constants
from rasa_core.utils import EndpointConfig

logger = logging.getLogger(__name__)

INTENT_MESSAGE_PREFIX = "/"


class NaturalLanguageInterpreter(object):
    def parse(self, text):
        raise NotImplementedError(
                "Interpreter needs to be able to parse "
                "messages into structured output.")

    @staticmethod
    def create(obj):
        if isinstance(obj, NaturalLanguageInterpreter):
            return obj
        if isinstance(obj, str):
            return RasaNLUInterpreter(model_directory=obj)
        else:
            return RegexInterpreter()  # default interpreter


class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX + "_"   # _ is deprecated but supported

    @staticmethod
    def _create_entities(parsed_entities, sidx, eidx):
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append({
                    "entity": k,
                    "start": sidx,
                    "end": eidx,  # can't be more specific
                    "value": value
                })
        return entities

    @staticmethod
    def _parse_parameters(entitiy_str, sidx, eidx, user_input):
        # type: (Text, int, int, Text) -> List[Dict[Text, Any]]
        if entitiy_str is None or not entitiy_str.strip():
            # if there is nothing to parse we will directly exit
            return []

        try:
            parsed_entities = json.loads(entitiy_str)
            if isinstance(parsed_entities, dict):
                return RegexInterpreter._create_entities(parsed_entities,
                                                         sidx, eidx)
            else:
                raise Exception("Parsed value isn't a json object "
                                "(instead parser found '{}')"
                                ".".format(type(parsed_entities)))
        except Exception as e:
            logger.warning("Invalid to parse arguments in line "
                           "'{}'. Failed to decode parameters"
                           "as a json object. Make sure the intent"
                           "followed by a proper json object. "
                           "Error: {}".format(user_input, e))
            return []

    @staticmethod
    def _parse_confidence(confidence_str):
        # type: (Text) -> float
        if confidence_str is None:
            return 1.0

        try:
            return float(confidence_str.strip()[1:])
        except Exception as e:
            logger.warning("Invalid to parse confidence value in line "
                           "'{}'. Make sure the intent confidence is an "
                           "@ followed by a decimal number. "
                           "Error: {}".format(confidence_str, e))
            return 0.0

    @staticmethod
    def extract_intent_and_entities(user_input):
        # type: (Text) -> object
        """Parse the user input using regexes to extract intent & entities."""

        prefixes = re.escape(RegexInterpreter.allowed_prefixes())
{'fraction_non_alphanumeric': 0.07561660020676414, 'fraction_numerical': 0.0011815093782306897, 'mean_word_length': 1.868276154171961, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415330', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1682, 'n_words': 529}