def train(self, training_data: TrainingData, config: Optional[RasaNLUModelConfig]=None, **kwargs: Any) -> None:
    'Tokenize all training data.'
    for example in training_data.training_examples:
        for attribute in MESSAGE_ATTRIBUTES:
            if (example.get(attribute) is not None):
                if (attribute == INTENT_ATTRIBUTE):
                    tokens = self._split_intent(example)
                else:
                    tokens = self.tokenize(example, attribute)
                    tokens = self.add_cls_token(tokens, attribute)
                example.set(TOKENS_NAMES[attribute], tokens)
Tokenize all training data.
def train(self, training_data: TrainingData, config: Optional[RasaNLUModelConfig]=None, **kwargs: Any) -> None:
    
    for example in training_data.training_examples:
        for attribute in MESSAGE_ATTRIBUTES:
            if (example.get(attribute) is not None):
                if (attribute == INTENT_ATTRIBUTE):
                    tokens = self._split_intent(example)
                else:
                    tokens = self.tokenize(example, attribute)
                    tokens = self.add_cls_token(tokens, attribute)
                example.set(TOKENS_NAMES[attribute], tokens)