<commit_before>from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Text, Tuple, Type
import numpy as np
import scipy.sparse

import rasa.shared.utils.io
import rasa.utils.io
import rasa.nlu.utils.pattern_utils as pattern_utils
from rasa.engine.graph import ExecutionContext, GraphComponent
from rasa.engine.storage.resource import Resource
from rasa.engine.storage.storage import ModelStorage
from rasa.nlu.constants import TOKENS_NAMES
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer2
from rasa.nlu.tokenizers.tokenizer import Tokenizer
from rasa.shared.nlu.constants import (
    TEXT,
    RESPONSE,
    ACTION_TEXT,
)
from rasa.shared.nlu.training_data.training_data import TrainingData
from rasa.shared.nlu.training_data.message import Message
from rasa.nlu.featurizers.sparse_featurizer._regex_featurizer import RegexFeaturizer

logger = logging.getLogger(__name__)

# TODO: remove after all references to old featurizer have been removed
RegexFeaturizer = RegexFeaturizer


class RegexFeaturizerGraphComponent(SparseFeaturizer2, GraphComponent):
    """Adds message features based on regex expressions."""

    @staticmethod
    def get_default_config() -> Dict[Text, Any]:
        """Returns the component's default config."""
        return {
            **SparseFeaturizer2.get_default_config(),
            # text will be processed with case sensitive as default
            "case_sensitive": True,
            # use lookup tables to generate features
            "use_lookup_tables": True,
            # use regexes to generate features
            "use_regexes": True,
            # use match word boundaries for lookup table
            "use_word_boundaries": True,
            # Additional number of patterns to consider
            # for incremental training
            "number_additional_patterns": None,
        }

    def __init__(
        self,
        config: Dict[Text, Any],
        model_storage: ModelStorage,
        resource: Resource,
        execution_context: ExecutionContext,
        known_patterns: Optional[List[Dict[Text, Text]]] = None,
    ) -> None:
        """Constructs new features for regexes and lookup table using regex expressions.

        Args:
            config: Configuration for the component.
            model_storage: Storage which graph components can use to persist and load
                themselves.
            resource: Resource locator for this component which can be used to persist
                and load itself from the `model_storage`.
            execution_context: Information about the current graph run.
            known_patterns: Regex Patterns the component should pre-load itself with.
        """
        super().__init__(execution_context.node_name, config)

        self._model_storage = model_storage
        self._resource = resource

        self.known_patterns = known_patterns if known_patterns else []
        self.case_sensitive = config["case_sensitive"]
        self.finetune_mode = execution_context.is_finetuning

        if config["number_additional_patterns"]:
            rasa.shared.utils.io.raise_deprecation_warning(
                "The parameter `number_additional_patterns` has been deprecated "
                "since the pipeline does not create an extra buffer for new vocabulary "
                "anymore. Any value assigned to this parameter will be ignored. "
                "You can omit specifying `number_additional_patterns` in future runs."
            )

    def _merge_new_patterns(self, new_patterns: List[Dict[Text, Text]]) -> None:
        """Updates already known patterns with new patterns extracted from data.

        New patterns should always be added to the end of the existing
        patterns and the order of the existing patterns should not be disturbed.

        Args:
            new_patterns: Patterns extracted from training data and to be merged with
                known patterns.
        """
        pattern_name_index_map = {
            pattern["name"]: index for index, pattern in enumerate(self.known_patterns)
        }
        for extra_pattern in new_patterns:
            new_pattern_name = extra_pattern["name"]
<commit_msg>add missing 'create'<commit_after>from __future__ import annotations
import logging
import re
from typing import Any, Dict, List, Optional, Text, Tuple, Type
import numpy as np
import scipy.sparse

import rasa.shared.utils.io
import rasa.utils.io
import rasa.nlu.utils.pattern_utils as pattern_utils
from rasa.engine.graph import ExecutionContext, GraphComponent
from rasa.engine.storage.resource import Resource
from rasa.engine.storage.storage import ModelStorage
from rasa.nlu.constants import TOKENS_NAMES
from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer2
from rasa.nlu.tokenizers.tokenizer import Tokenizer
from rasa.shared.nlu.constants import (
    TEXT,
    RESPONSE,
    ACTION_TEXT,
)
from rasa.shared.nlu.training_data.training_data import TrainingData
from rasa.shared.nlu.training_data.message import Message
from rasa.nlu.featurizers.sparse_featurizer._regex_featurizer import RegexFeaturizer

logger = logging.getLogger(__name__)

# TODO: remove after all references to old featurizer have been removed
RegexFeaturizer = RegexFeaturizer


class RegexFeaturizerGraphComponent(SparseFeaturizer2, GraphComponent):
    """Adds message features based on regex expressions."""

    @staticmethod
    def get_default_config() -> Dict[Text, Any]:
        """Returns the component's default config."""
        return {
            **SparseFeaturizer2.get_default_config(),
            # text will be processed with case sensitive as default
            "case_sensitive": True,
            # use lookup tables to generate features
            "use_lookup_tables": True,
            # use regexes to generate features
            "use_regexes": True,
            # use match word boundaries for lookup table
            "use_word_boundaries": True,
            # Additional number of patterns to consider
            # for incremental training
            "number_additional_patterns": None,
        }

    def __init__(
        self,
        config: Dict[Text, Any],
        model_storage: ModelStorage,
        resource: Resource,
        execution_context: ExecutionContext,
        known_patterns: Optional[List[Dict[Text, Text]]] = None,
    ) -> None:
        """Constructs new features for regexes and lookup table using regex expressions.

        Args:
            config: Configuration for the component.
            model_storage: Storage which graph components can use to persist and load
                themselves.
            resource: Resource locator for this component which can be used to persist
                and load itself from the `model_storage`.
            execution_context: Information about the current graph run.
            known_patterns: Regex Patterns the component should pre-load itself with.
        """
        super().__init__(execution_context.node_name, config)

        self._model_storage = model_storage
        self._resource = resource

        self.known_patterns = known_patterns if known_patterns else []
        self.case_sensitive = config["case_sensitive"]
        self.finetune_mode = execution_context.is_finetuning

        if config["number_additional_patterns"]:
            rasa.shared.utils.io.raise_deprecation_warning(
                "The parameter `number_additional_patterns` has been deprecated "
                "since the pipeline does not create an extra buffer for new vocabulary "
                "anymore. Any value assigned to this parameter will be ignored. "
                "You can omit specifying `number_additional_patterns` in future runs."
            )

    @classmethod
    def create(
        cls,
        config: Dict[Text, Any],
        model_storage: ModelStorage,
        resource: Resource,
        execution_context: ExecutionContext,
    ) -> RegexFeaturizerGraphComponent:
        """Creates a new untrained component (see parent class for full docstring)."""
        return cls(config, model_storage, resource, execution_context)

    def _merge_new_patterns(self, new_patterns: List[Dict[Text, Text]]) -> None:
        """Updates already known patterns with new patterns extracted from data.

        New patterns should always be added to the end of the existing
        patterns and the order of the existing patterns should not be disturbed.

        Args:
            new_patterns: Patterns extracted from training data and to be merged with
                known patterns.
        """
        pattern_name_index_map = {
            pattern["name"]: index for index, pattern in enumerate(self.known_patterns)
        }
        for extra_pattern in new_patterns:
            new_pattern_name = extra_pattern["name"]

{'fraction_non_alphanumeric': 0.0630256812829427, 'fraction_numerical': 0.000672872042166648, 'mean_word_length': 2.834049871023216, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '825499', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2260, 'n_words': 752}