





from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers.models.distilbert.modeling_tf_distilbert import (
         TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFDistilBertForMaskedLM,
         TFDistilBertForMultipleChoice,
         TFDistilBertForQuestionAnswering,
         TFDistilBertForSequenceClassification,
         TFDistilBertForTokenClassification,
         TFDistilBertModel,
     )







class 				__lowerCAmelCase       :







      def __init__(    self		,      lowerCAmelCase__		,      )   ->						int:



        '''simple docstring'''




        a__	:					Optional[int]										=parent
        a__	:					Union[str, Any]										=1_3
        a__	:					Tuple										=7
        a__	:					List[str]										=True
        a__	:					Union[str, Any]										=True
        a__	:					Optional[Any]										=False
        a__	:					List[Any]										=True
        a__	:					Optional[int]										=9_9
        a__	:					Any										=3_2
        a__	:					List[str]										=2
        a__	:					List[Any]										=4
        a__	:					List[Any]										=3_7
        a__	:					Tuple										="gelu"
        a__	:					Optional[Any]										=0.1
        a__	:					Dict										=0.1
        a__	:					int										=5_1_2
        a__	:					str										=1_6
        a__	:					List[Any]										=2
        a__	:					int										=0.02
        a__	:					List[Any]										=3
        a__	:					Any										=4
        a__	:					Optional[int]										=None







      def    _lowercase  (    self							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					Optional[int]										=ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size							)

        a__	:					str										=None
        if self.use_input_mask:
          a__	:					Optional[Any]										=random_attention_mask([self.batch_size, self.seq_length]							)

        a__	:					List[str]										=None
        a__	:					str										=None
        a__	:					Dict										=None
        if self.use_labels:
          a__	:					Optional[int]										=ids_tensor([self.batch_size]		,      self.type_sequence_label_size							)
          a__	:					List[Any]										=ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels							)
          a__	:					Any										=ids_tensor([self.batch_size]		,      self.num_choices							)

        a__	:					List[str]										=DistilBertConfig(
            vocab_size=self.vocab_size		,      dim=self.hidden_size		,      n_layers=self.num_hidden_layers		,      n_heads=self.num_attention_heads		,      hidden_dim=self.intermediate_size		,      hidden_act=self.hidden_act		,      dropout=self.hidden_dropout_prob		,      attention_dropout=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      initializer_range=self.initializer_range		,      )

        return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					int										=TFDistilBertModel(config=lowerCAmelCase__							)
        a__	:					Tuple										={"input_ids": input_ids, "attention_mask": input_mask}

        a__	:					List[Any]										=model(lowerCAmelCase__							)

        a__	:					Tuple										=[input_ids, input_mask]

        a__	:					Tuple										=model(lowerCAmelCase__							)

        self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size)							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=TFDistilBertForMaskedLM(config=lowerCAmelCase__							)
        a__	:					Dict										={"input_ids": input_ids, "attention_mask": input_mask}
        a__	:					List[str]										=model(lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						List[Any]:



        '''simple docstring'''




        a__	:					List[str]										=TFDistilBertForQuestionAnswering(config=lowerCAmelCase__							)
        a__	:					Union[str, Any]										={
            "input_ids": input_ids,
            "attention_mask": input_mask,
        }
        a__	:					int										=model(lowerCAmelCase__							)
        self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length)							)
        self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length)							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					Tuple										=self.num_labels
        a__	:					Optional[int]										=TFDistilBertForSequenceClassification(lowerCAmelCase__							)
        a__	:					List[Any]										={"input_ids": input_ids, "attention_mask": input_mask}
        a__	:					List[Any]										=model(lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=self.num_choices
        a__	:					Dict										=TFDistilBertForMultipleChoice(lowerCAmelCase__							)
        a__	:					Dict										=tf.tile(tf.expand_dims(lowerCAmelCase__		,      1							)		,      (1, self.num_choices, 1)							)
        a__	:					Optional[int]										=tf.tile(tf.expand_dims(lowerCAmelCase__		,      1							)		,      (1, self.num_choices, 1)							)
        a__	:					Optional[int]										={
            "input_ids": multiple_choice_inputs_ids,
            "attention_mask": multiple_choice_input_mask,
        }
        a__	:					int										=model(lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices)							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        a__	:					Optional[int]										=self.num_labels
        a__	:					Dict										=TFDistilBertForTokenClassification(lowerCAmelCase__							)
        a__	:					str										={"input_ids": input_ids, "attention_mask": input_mask}
        a__	:					Tuple										=model(lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels)							)







      def    _lowercase  (    self							)   ->						Dict:



        '''simple docstring'''




        a__	:					Dict										=self.prepare_config_and_inputs()
        ((a__)					,							(a__)					,							(a__)					,							(a__)					,							(a__)					,							(a__))	:					Dict										=config_and_inputs
        a__	:					List[str]										={"input_ids": input_ids, "attention_mask": input_mask}
        return config, inputs_dict







@require_tf
class 				__lowerCAmelCase       (    UpperCamelCase__						,				UpperCamelCase__						,				unittest.TestCase):
      _lowercase				:					Any						=			(
          (
              TFDistilBertModel,
              TFDistilBertForMaskedLM,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertForMultipleChoice,
          )
          if is_tf_available()
          else None
      )
      _lowercase				:					List[Any]						=			(
          {
              """feature-extraction""": TFDistilBertModel,
              """fill-mask""": TFDistilBertForMaskedLM,
              """question-answering""": TFDistilBertForQuestionAnswering,
              """text-classification""": TFDistilBertForSequenceClassification,
              """token-classification""": TFDistilBertForTokenClassification,
              """zero-shot""": TFDistilBertForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      _lowercase				:					int						=			False
      _lowercase				:					List[str]						=			False







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__	:					Dict										=TFDistilBertModelTester(self							)
        a__	:					Optional[int]										=ConfigTester(self		,      config_class=lowerCAmelCase__		,      dim=3_7							)







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__	:					int										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_model(*lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Optional[int]										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_masked_lm(*lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Dict										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_question_answering(*lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        a__	:					List[str]										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					int										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_token_classification(*lowerCAmelCase__							)







      @slow
      def    _lowercase  (    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]							):
          a__	:					List[str]										=TFDistilBertModel.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)







@require_tf
class 				__lowerCAmelCase       (    unittest.TestCase):







      @slow
      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Dict										=TFDistilBertModel.from_pretrained("distilbert-base-uncased"							)
        a__	:					Optional[int]										=tf.constant([[0, 1, 2, 3, 4, 5]]							)
        a__	:					Tuple										=model(lowerCAmelCase__							)[0]

        a__	:					List[str]										=[1, 6, 7_6_8]
        self.assertEqual(output.shape		,      lowerCAmelCase__							)

        a__	:					List[Any]										=tf.constant(
            [
                [
                    [0.19_26_18_85, -0.13_73_29_55, 0.4_11_97_99],
                    [0.22_15_01_56, -0.07_42_26_61, 0.39_03_72_04],
                    [0.22_75_60_18, -0.0_89_64_14, 0.3_70_14_67],
                ]
            ]							)
        tf.debugging.assert_near(output[:, :3, :3]		,      lowerCAmelCase__		,      atol=1E-4							)













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase	:	int		     =       logging.get_logger(__name__)

UpperCAmelCase	:	Optional[Any]		     =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase	:	List[Any]		     =       [
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

UpperCAmelCase	:	Optional[int]		     =       {
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					str						=			VOCAB_FILES_NAMES
      _lowercase				:					List[Any]						=			PRETRAINED_VOCAB_FILES_MAP
      _lowercase				:					Dict						=			PRETRAINED_INIT_CONFIGURATION
      _lowercase				:					Union[str, Any]						=			FunnelTokenizer
      _lowercase				:					Any						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase				:					int						=			2







      def __init__(    self		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=True		,      lowerCAmelCase__="<unk>"		,      lowerCAmelCase__="<sep>"		,      lowerCAmelCase__="<pad>"		,      lowerCAmelCase__="<cls>"		,      lowerCAmelCase__="<mask>"		,      lowerCAmelCase__="<s>"		,      lowerCAmelCase__="</s>"		,      lowerCAmelCase__=True		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      lowerCAmelCase__="##"		,      **lowerCAmelCase__		,      )   ->						Optional[int]:



        '''simple docstring'''




        super().__init__(
            lowerCAmelCase__		,      tokenizer_file=lowerCAmelCase__		,      do_lower_case=lowerCAmelCase__		,      unk_token=lowerCAmelCase__		,      sep_token=lowerCAmelCase__		,      pad_token=lowerCAmelCase__		,      cls_token=lowerCAmelCase__		,      mask_token=lowerCAmelCase__		,      bos_token=lowerCAmelCase__		,      eos_token=lowerCAmelCase__		,      clean_text=lowerCAmelCase__		,      tokenize_chinese_chars=lowerCAmelCase__		,      strip_accents=lowerCAmelCase__		,      wordpieces_prefix=lowerCAmelCase__		,      **lowerCAmelCase__		,      )

        a__	:					Optional[Any]										=json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
        if (
            normalizer_state.get("lowercase"		,      lowerCAmelCase__							) != do_lower_case
            or normalizer_state.get("strip_accents"		,      lowerCAmelCase__							) != strip_accents
            or normalizer_state.get("handle_chinese_chars"		,      lowerCAmelCase__							) != tokenize_chinese_chars
        ):
          a__	:					List[str]										=getattr(lowerCAmelCase__		,      normalizer_state.pop("type"							)							)
          a__	:					Union[str, Any]										=do_lower_case
          a__	:					Any										=strip_accents
          a__	:					Optional[Any]										=tokenize_chinese_chars
          a__	:					Dict										=normalizer_class(**lowerCAmelCase__							)

        a__	:					Any										=do_lower_case







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__=None							)   ->						str:



        '''simple docstring'''




        a__	:					Dict										=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						List[int]:



        '''simple docstring'''




        a__	:					Optional[int]										=[self.sep_token_id]
        a__	:					Union[str, Any]										=[self.cls_token_id]
        if token_ids_a is None:
          return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0]
        return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						Tuple[str]:



        '''simple docstring'''




        a__	:					Tuple										=self._tokenizer.model.save(lowerCAmelCase__		,      name=lowerCAmelCase__							)
        return tuple(lowerCAmelCase__							)






