

"""simple docstring"""


import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM





@require_torch
class lowerCamelCase__   :



    '''simple docstring'''




    def __init__( self ,lowerCamelCase_ ,lowerCamelCase_=9_9 ,lowerCamelCase_=1_3 ,lowerCamelCase_=1_6 ,lowerCamelCase_=7 ,lowerCamelCase_=True ,lowerCamelCase_=True ,lowerCamelCase_=True ,lowerCamelCase_=False ,lowerCamelCase_=True ,lowerCamelCase_=2 ,lowerCamelCase_=3_2 ,lowerCamelCase_=4 ,lowerCamelCase_=4 ,lowerCamelCase_=3_0 ,lowerCamelCase_=0 ,lowerCamelCase_=1 ,lowerCamelCase_=2 ,lowerCamelCase_=None ,)    ->      Any:
          A				  =		parent
          A				  =		batch_size
          A				  =		decoder_seq_length
          # For common tests
          A				  =		self.decoder_seq_length
          A				  =		is_training
          A				  =		use_attention_mask
          A				  =		use_labels

          A				  =		vocab_size
          A				  =		d_model
          A				  =		d_model
          A				  =		decoder_layers
          A				  =		decoder_layers
          A				  =		decoder_ffn_dim
          A				  =		decoder_attention_heads
          A				  =		decoder_attention_heads
          A				  =		eos_token_id
          A				  =		bos_token_id
          A				  =		pad_token_id
          A				  =		decoder_start_token_id
          A				  =		use_cache
          A				  =		max_position_embeddings

          A				  =		None
          A				  =		decoder_seq_length
          A				  =		2
          A				  =		1




    def UpperCamelCase__		( self    )    ->      List[str]:
          A				  =		ids_tensor([self.batch_size, self.decoder_seq_length] ,self.vocab_size    )

          A				  =		None
          if self.use_attention_mask:
                A				  =		ids_tensor([self.batch_size, self.decoder_seq_length] ,vocab_size=2    )

          A				  =		None
          if self.use_labels:
                A				  =		ids_tensor([self.batch_size, self.decoder_seq_length] ,self.vocab_size    )

          A				  =		TrOCRConfig(
              vocab_size=self.vocab_size ,d_model=self.d_model ,decoder_layers=self.decoder_layers ,decoder_ffn_dim=self.decoder_ffn_dim ,decoder_attention_heads=self.decoder_attention_heads ,eos_token_id=self.eos_token_id ,bos_token_id=self.bos_token_id ,use_cache=self.use_cache ,pad_token_id=self.pad_token_id ,decoder_start_token_id=self.decoder_start_token_id ,max_position_embeddings=self.max_position_embeddings ,)

          return (config, input_ids, attention_mask, lm_labels)




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_ ,)    ->      Union[str, Any]:
          A				  =		True
          A				  =		TrOCRDecoder(config=lowerCamelCase_    ).to(lowerCamelCase_    ).eval()
          A				  =		input_ids[:2]

          input_ids[input_ids == 0] += 1
          # first forward pass
          A				  =		model(lowerCamelCase_ ,use_cache=lowerCamelCase_    )
          A				  =		model(lowerCamelCase_    )
          A				  =		model(lowerCamelCase_ ,use_cache=lowerCamelCase_    )

          self.parent.assertTrue(len(lowerCamelCase_    ) == len(lowerCamelCase_    )    )
          self.parent.assertTrue(len(lowerCamelCase_    ) == len(lowerCamelCase_    ) + 1    )

          A				  =		outputs["""past_key_values"""]

          # create hypothetical next token and extent to next_input_ids
          A				  =		ids_tensor((2, 1) ,config.vocab_size - 1    ) + 1

          # append to next input_ids and
          A				  =		torch.cat([input_ids, next_tokens] ,dim=-1    )

          A				  =		model(lowerCamelCase_    )["""last_hidden_state"""]
          A				  =		model(lowerCamelCase_ ,past_key_values=lowerCamelCase_    )["""last_hidden_state"""]

          # select random slice
          A				  =		ids_tensor((1,) ,output_from_past.shape[-1]    ).item()
          A				  =		output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
          A				  =		output_from_past[:, 0, random_slice_idx].detach()

          # test that outputs are equal for slice
          assert torch.allclose(lowerCamelCase_ ,lowerCamelCase_ ,atol=1E-3    )




    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          A				  =		self.prepare_config_and_inputs()
          A  ,			A  ,			A  ,			A				  =		config_and_inputs

          A				  =		{"""input_ids""": input_ids, """attention_mask""": attention_mask}
          return config, inputs_dict





@require_torch
class lowerCamelCase__   (				SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE     ,   SCREAMING_SNAKE_CASE     ,   unittest.TestCase    ):



    '''simple docstring'''




    _lowerCamelCase					=   (TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
    _lowerCamelCase					=   (TrOCRForCausalLM,) if is_torch_available() else ()
    _lowerCamelCase					=   {'''text-generation''': TrOCRForCausalLM} if is_torch_available() else {}
    _lowerCamelCase					=   True
    _lowerCamelCase					=   False




    def UpperCamelCase__		( self    )    ->      int:
          A				  =		TrOCRStandaloneDecoderModelTester(self ,is_training=lowerCamelCase_    )
          A				  =		ConfigTester(self ,config_class=lowerCamelCase_    )




    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          pass




    def UpperCamelCase__		( self    )    ->      Tuple:
          pass




    def UpperCamelCase__		( self    )    ->      Any:
          pass




    def UpperCamelCase__		( self    )    ->      Any:
          self.config_tester.run_common_tests()




    def UpperCamelCase__		( self    )    ->      List[Any]:
          A				  =		self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_decoder_model_past(*lowerCamelCase_    )




    def UpperCamelCase__		( self    )    ->      Union[str, Any]:
          return




    @unittest.skip("""The model doesn't support left padding"""    )  # and it's not used enough to be worth fixing :)
    def UpperCamelCase__		( self    )    ->      List[Any]:
          pass







"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase													=logging.get_logger(__name__)

UpperCAmelCase													={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase													=[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCAmelCase													={
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCAmelCase													={f"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase													={f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   VOCAB_FILES_NAMES
    _lowerCamelCase					=   PRETRAINED_VOCAB_FILES_MAP
    _lowerCamelCase					=   PRETRAINED_INIT_CONFIGURATION
    _lowerCamelCase					=   FunnelTokenizer
    _lowerCamelCase					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCamelCase					=   2




    def __init__( self ,lowerCamelCase_=None ,lowerCamelCase_=None ,lowerCamelCase_=True ,lowerCamelCase_="<unk>" ,lowerCamelCase_="<sep>" ,lowerCamelCase_="<pad>" ,lowerCamelCase_="<cls>" ,lowerCamelCase_="<mask>" ,lowerCamelCase_="<s>" ,lowerCamelCase_="</s>" ,lowerCamelCase_=True ,lowerCamelCase_=True ,lowerCamelCase_=None ,lowerCamelCase_="##" ,**lowerCamelCase_ ,)    ->      List[Any]:
          super().__init__(
              lowerCamelCase_ ,tokenizer_file=lowerCamelCase_ ,do_lower_case=lowerCamelCase_ ,unk_token=lowerCamelCase_ ,sep_token=lowerCamelCase_ ,pad_token=lowerCamelCase_ ,cls_token=lowerCamelCase_ ,mask_token=lowerCamelCase_ ,bos_token=lowerCamelCase_ ,eos_token=lowerCamelCase_ ,clean_text=lowerCamelCase_ ,tokenize_chinese_chars=lowerCamelCase_ ,strip_accents=lowerCamelCase_ ,wordpieces_prefix=lowerCamelCase_ ,**lowerCamelCase_ ,)

          A				  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
          if (
              normalizer_state.get("""lowercase""" ,lowerCamelCase_    ) != do_lower_case
              or normalizer_state.get("""strip_accents""" ,lowerCamelCase_    ) != strip_accents
              or normalizer_state.get("""handle_chinese_chars""" ,lowerCamelCase_    ) != tokenize_chinese_chars
          ):
                A				  =		getattr(lowerCamelCase_ ,normalizer_state.pop("""type"""    )    )
                A				  =		do_lower_case
                A				  =		strip_accents
                A				  =		tokenize_chinese_chars
                A				  =		normalizer_class(**lowerCamelCase_    )

          A				  =		do_lower_case




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None    )    ->      List[Any]:
          A				  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      List[int]:
          A				  =		[self.sep_token_id]
          A				  =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0]
          return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      Tuple[str]:
          A				  =		self._tokenizer.model.save(lowerCamelCase_ ,name=lowerCamelCase_    )
          return tuple(lowerCamelCase_    )




