<commit_before>from mitie import named_entity_extractor
import cloudpickle
from rasa_nlu import Interpreter
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer


class MITIESklearnInterpreter(Interpreter):
    def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs):
        if entity_extractor_file:
            self.extractor = named_entity_extractor(entity_extractor_file)  # ,metadata["feature_extractor"])
        with open(intent_classifier_file, 'rb') as f:
            self.classifier = cloudpickle.load(f)
        self.featurizer = MITIEFeaturizer(feature_extractor_file)
        self.tokenizer = MITIETokenizer()

    def get_entities(self, tokens):
        d = {}
        entities = self.extractor.extract_entities(tokens)
        for e in entities:
            _range = e[0]
            d[e[1]] = " ".join(tokens[i] for i in _range)
        return d

    def get_intent(self, tokens):
        """Returns the most likely intent and its probability for the input text.

        :param text: text to classify
        :return: tuple of most likely intent name and its probability"""
        if self.classifier:
            X = self.featurizer.create_bow_vecs(tokens)
            intent_ids, probabilities = self.classifier.predict(X)
            intents = self.classifier.transform_labels_num2str(intent_ids)
            intent, score = intents[0], probabilities[0]<commit_msg>Replace tokens with text in function parameters<commit_after>from mitie import named_entity_extractor
import cloudpickle
from rasa_nlu import Interpreter
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer


class MITIESklearnInterpreter(Interpreter):
    def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs):
        if entity_extractor_file:
            self.extractor = named_entity_extractor(entity_extractor_file)  # ,metadata["feature_extractor"])
        with open(intent_classifier_file, 'rb') as f:
            self.classifier = cloudpickle.load(f)
        self.featurizer = MITIEFeaturizer(feature_extractor_file)
        self.tokenizer = MITIETokenizer()

    def get_entities(self, tokens):
        d = {}
        entities = self.extractor.extract_entities(tokens)
        for e in entities:
            _range = e[0]
            d[e[1]] = " ".join(tokens[i] for i in _range)
        return d

    def get_intent(self, text):
        """Returns the most likely intent and its probability for the input text.

        :param text: text to classify
        :return: tuple of most likely intent name and its probability"""
        if self.classifier:
            X = self.featurizer.create_bow_vecs(text)
            intent_ids, probabilities = self.classifier.predict(X)
            intents = self.classifier.transform_labels_num2str(intent_ids)
            intent, score = intents[0], probabilities[0]
{'fraction_non_alphanumeric': 0.07877604166666667, 'fraction_numerical': 0.0032552083333333335, 'mean_word_length': 3.3588652482269503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3511843', 'n_tokens_mistral': 908, 'n_tokens_neox': 905, 'n_words': 215}