<commit_before>import time
from collections import Counter

import logging
import numpy as np
import os
import typing
import warnings
from string import punctuation
from typing import Any, Dict, List, Optional, Text

from rasa.nlu import utils
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.featurizers import Featurizer
from rasa.nlu.training_data import Message, TrainingData
from rasa.nlu.utils import write_json_to_file

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.model import Metadata


class NGramFeaturizer(Featurizer):

    provides = ["text_features"]

    requires = ["spacy_doc"]

    defaults = {
        # defines the maximum number of ngrams to collect and add
        # to the featurization of a sentence
        "max_number_of_ngrams": 10,
        # the minimal length in characters of an ngram to be eligible
        "ngram_min_length": 3,
        # the maximal length in characters of an ngram to be eligible
        "ngram_max_length": 17,
        # the minimal number of times an ngram needs to occur in the
        # training data to be considered as a feature
        "ngram_min_occurrences": 5,
        # during cross validation (used to detect which ngrams are most
        # valuable) every intent with fever examples than this config
        # value will be excluded
        "min_intent_examples": 4,
    }

    def __init__(self,
                 component_config: Optional[Dict[Text, Any]] = None,
                 all_ngrams: Optional[List[Text]] = None,
                 best_num_ngrams: Optional[int] = None):
        super(NGramFeaturizer, self).__init__(component_config)

        self.best_num_ngrams = best_num_ngrams
        self.all_ngrams = all_ngrams

    @classmethod
    def required_packages(cls) -> List[Text]:
        return ["spacy", "sklearn", "cloudpickle"]

    def train(
        self, training_data: TrainingData, cfg: RasaNLUModelConfig, **kwargs: Any
    ):

        start = time.time()
        self.train_on_sentences(training_data.intent_examples)
        logger.debug("Ngram collection took {} seconds".format(time.time() - start))

        for example in training_data.training_examples:
            updated = self._text_features_with_ngrams(example, self.best_num_ngrams)<commit_msg>fixed black<commit_after>import time
from collections import Counter

import logging
import numpy as np
import os
import typing
import warnings
from string import punctuation
from typing import Any, Dict, List, Optional, Text

from rasa.nlu import utils
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.featurizers import Featurizer
from rasa.nlu.training_data import Message, TrainingData
from rasa.nlu.utils import write_json_to_file

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.model import Metadata


class NGramFeaturizer(Featurizer):

    provides = ["text_features"]

    requires = ["spacy_doc"]

    defaults = {
        # defines the maximum number of ngrams to collect and add
        # to the featurization of a sentence
        "max_number_of_ngrams": 10,
        # the minimal length in characters of an ngram to be eligible
        "ngram_min_length": 3,
        # the maximal length in characters of an ngram to be eligible
        "ngram_max_length": 17,
        # the minimal number of times an ngram needs to occur in the
        # training data to be considered as a feature
        "ngram_min_occurrences": 5,
        # during cross validation (used to detect which ngrams are most
        # valuable) every intent with fever examples than this config
        # value will be excluded
        "min_intent_examples": 4,
    }

    def __init__(
        self,
        component_config: Optional[Dict[Text, Any]] = None,
        all_ngrams: Optional[List[Text]] = None,
        best_num_ngrams: Optional[int] = None,
    ):
        super(NGramFeaturizer, self).__init__(component_config)

        self.best_num_ngrams = best_num_ngrams
        self.all_ngrams = all_ngrams

    @classmethod
    def required_packages(cls) -> List[Text]:
        return ["spacy", "sklearn", "cloudpickle"]

    def train(
        self, training_data: TrainingData, cfg: RasaNLUModelConfig, **kwargs: Any
    ):

        start = time.time()
        self.train_on_sentences(training_data.intent_examples)
        logger.debug("Ngram collection took {} seconds".format(time.time() - start))

        for example in training_data.training_examples:
            updated = self._text_features_with_ngrams(example, self.best_num_ngrams)
{'fraction_non_alphanumeric': 0.07199124726477024, 'fraction_numerical': 0.003063457330415755, 'mean_word_length': 3.1143114311431144, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1070392', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1291, 'n_words': 415}