



"""simple docstring"""







import inspect
import unittest

from transformers import YolosConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import YolosForObjectDetection, YolosModel
							from transformers.models.yolos.modeling_yolos import YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor
class 			__A			:







		'''simple docstring'''





		def __init__(      self		:    List[str]     ,_snake_case		:    List[str]     ,_snake_case		:    int=13     ,_snake_case		:    str=[30, 30]     ,_snake_case		:    Union[str, Any]=2     ,_snake_case		:    Tuple=3     ,_snake_case		:    Tuple=True     ,_snake_case		:    Optional[int]=True     ,_snake_case		:    Dict=32     ,_snake_case		:    Union[str, Any]=5     ,_snake_case		:    Tuple=4     ,_snake_case		:    Optional[int]=37     ,_snake_case		:    Optional[Any]="gelu"     ,_snake_case		:    List[str]=0.1     ,_snake_case		:    str=0.1     ,_snake_case		:    int=10     ,_snake_case		:    Union[str, Any]=0.02     ,_snake_case		:    Optional[Any]=3     ,_snake_case		:    Optional[int]=None     ,_snake_case		:    int=8     ,_snake_case		:    List[Any]=10     ,)   ->						Optional[Any]:






				"""simple docstring"""



				lowercase__  :       Union[str, Any]												=     parent
				lowercase__  :       Dict												=     batch_size
				lowercase__  :       Dict												=     image_size
				lowercase__  :       str												=     patch_size
				lowercase__  :       List[Any]												=     num_channels
				lowercase__  :       Dict												=     is_training
				lowercase__  :       Any												=     use_labels
				lowercase__  :       List[Any]												=     hidden_size
				lowercase__  :       Dict												=     num_hidden_layers
				lowercase__  :       int												=     num_attention_heads
				lowercase__  :       Optional[int]												=     intermediate_size
				lowercase__  :       int												=     hidden_act
				lowercase__  :       Tuple												=     hidden_dropout_prob
				lowercase__  :       Union[str, Any]												=     attention_probs_dropout_prob
				lowercase__  :       int												=     type_sequence_label_size
				lowercase__  :       Union[str, Any]												=     initializer_range
				lowercase__  :       Union[str, Any]												=     num_labels
				lowercase__  :       Tuple												=     scope
				lowercase__  :       Tuple												=     n_targets
				lowercase__  :       Dict												=     num_detection_tokens
				# we set the expected sequence length (which is used in several tests)
				# expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + num_detection_tokens
				lowercase__  :       Tuple												=     (image_size[1] // patch_size) * (image_size[0] // patch_size)
				lowercase__  :       Dict												=     num_patches + 1 + self.num_detection_tokens







		def    UpperCAmelCase       (      self		:    Union[str, Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				lowercase__  :       Optional[int]												=     floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]		)

				lowercase__  :       Union[str, Any]												=     None
				if self.use_labels:
						# labels is a list of Dict (each Dict being the labels for a given example in the batch)
						lowercase__  :       List[Any]												=     []
						for i in range(self.batch_size		):
								lowercase__  :       str												=     {}
								lowercase__  :       Union[str, Any]												=     torch.randint(
								    high=self.num_labels     ,size=(self.n_targets,)     ,device=__UpperCamelCase		)
								lowercase__  :       Tuple												=     torch.rand(self.n_targets     ,4     ,device=__UpperCamelCase		)
								labels.append(__UpperCamelCase		)

				lowercase__  :       Tuple												=     self.get_config()

				return config, pixel_values, labels







		def    UpperCAmelCase       (      self		:    Union[str, Any]		)   ->						List[Any]:






				"""simple docstring"""



				return YolosConfig(
				    image_size=self.image_size     ,patch_size=self.patch_size     ,num_channels=self.num_channels     ,hidden_size=self.hidden_size     ,num_hidden_layers=self.num_hidden_layers     ,num_attention_heads=self.num_attention_heads     ,intermediate_size=self.intermediate_size     ,hidden_act=self.hidden_act     ,hidden_dropout_prob=self.hidden_dropout_prob     ,attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,is_decoder=__UpperCamelCase     ,initializer_range=self.initializer_range     ,num_detection_tokens=self.num_detection_tokens     ,num_labels=self.num_labels     ,)







		def    UpperCAmelCase       (      self		:    Tuple     ,_snake_case		:    int     ,_snake_case		:    Union[str, Any]     ,_snake_case		:    str		)   ->						Union[str, Any]:






				"""simple docstring"""



				lowercase__  :       Dict												=     YolosModel(config=__UpperCamelCase		)
				model.to(__UpperCamelCase		)
				model.eval()
				lowercase__  :       Union[str, Any]												=     model(__UpperCamelCase		)
				self.parent.assertEqual(
				    result.last_hidden_state.shape     ,(self.batch_size, self.expected_seq_len, self.hidden_size)		)







		def    UpperCAmelCase       (      self		:    Tuple     ,_snake_case		:    Union[str, Any]     ,_snake_case		:    List[str]     ,_snake_case		:    Optional[Any]		)   ->						Tuple:






				"""simple docstring"""



				lowercase__  :       Union[str, Any]												=     YolosForObjectDetection(__UpperCamelCase		)
				model.to(__UpperCamelCase		)
				model.eval()

				lowercase__  :       Tuple												=     model(pixel_values=__UpperCamelCase		)
				lowercase__  :       str												=     model(__UpperCamelCase		)

				self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_detection_tokens, self.num_labels + 1)		)
				self.parent.assertEqual(result.pred_boxes.shape     ,(self.batch_size, self.num_detection_tokens, 4)		)

				lowercase__  :       Dict												=     model(pixel_values=__UpperCamelCase     ,labels=__UpperCamelCase		)

				self.parent.assertEqual(result.loss.shape     ,()		)
				self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_detection_tokens, self.num_labels + 1)		)
				self.parent.assertEqual(result.pred_boxes.shape     ,(self.batch_size, self.num_detection_tokens, 4)		)







		def    UpperCAmelCase       (      self		:    Tuple		)   ->						Any:






				"""simple docstring"""



				lowercase__  :       Any												=     self.prepare_config_and_inputs()
				lowercase__  :       Tuple												=     config_and_inputs
				lowercase__  :       str												=     {"""pixel_values""": pixel_values}
				return config, inputs_dict





@require_torch
class 			__A			(      __a	,__a	,unittest.TestCase       ):







		'''simple docstring'''





		lowerCAmelCase     :   str        =							(YolosModel, YolosForObjectDetection) if is_torch_available() else ()
		lowerCAmelCase     :   List[Any]        =							(
		    {'''feature-extraction''': YolosModel, '''object-detection''': YolosForObjectDetection} if is_torch_available() else {}
		)

		lowerCAmelCase     :   Optional[Any]        =							False
		lowerCAmelCase     :   Union[str, Any]        =							False
		lowerCAmelCase     :   Any        =							False
		lowerCAmelCase     :   List[Any]        =							False







		def    UpperCAmelCase       (      self		:    List[Any]     ,_snake_case		:    Dict     ,_snake_case		:    Dict     ,_snake_case		:    Tuple=False		)   ->						Union[str, Any]:






				"""simple docstring"""



				lowercase__  :       str												=     super()._prepare_for_class(__UpperCamelCase     ,__UpperCamelCase     ,return_labels=__UpperCamelCase		)

				if return_labels:
						if model_class.__name__ == "YolosForObjectDetection":
								lowercase__  :       str												=     []
								for i in range(self.model_tester.batch_size		):
										lowercase__  :       Optional[int]												=     {}
										lowercase__  :       Tuple												=     torch.ones(
										    size=(self.model_tester.n_targets,)     ,device=__UpperCamelCase     ,dtype=torch.long		)
										lowercase__  :       List[Any]												=     torch.ones(
										    self.model_tester.n_targets     ,4     ,device=__UpperCamelCase     ,dtype=torch.float		)
										labels.append(__UpperCamelCase		)
								lowercase__  :       Dict												=     labels

				return inputs_dict







		def    UpperCAmelCase       (      self		:    List[Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				lowercase__  :       List[Any]												=     YolosModelTester(self		)
				lowercase__  :       str												=     ConfigTester(self     ,config_class=__UpperCamelCase     ,has_text_modality=__UpperCamelCase     ,hidden_size=37		)







		def    UpperCAmelCase       (      self		:    Union[str, Any]		)   ->						List[str]:






				"""simple docstring"""



				self.config_tester.run_common_tests()







		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						Optional[int]:






				"""simple docstring"""



				pass







		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				lowercase__  :       str												=     self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						lowercase__  :       Optional[int]												=     model_class(__UpperCamelCase		)
						self.assertIsInstance(model.get_input_embeddings()     ,(nn.Module)		)
						lowercase__  :       Optional[Any]												=     model.get_output_embeddings()
						self.assertTrue(x is None or isinstance(__UpperCamelCase     ,nn.Linear		)		)







		def    UpperCAmelCase       (      self		:    Optional[int]		)   ->						int:






				"""simple docstring"""



				lowercase__  :       List[str]												=     self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						lowercase__  :       Dict												=     model_class(__UpperCamelCase		)
						lowercase__  :       List[Any]												=     inspect.signature(model.forward		)
						# signature.parameters is an OrderedDict => so arg_names order is deterministic
						lowercase__  :       Optional[int]												=     [*signature.parameters.keys()]

						lowercase__  :       str												=     ["""pixel_values"""]
						self.assertListEqual(arg_names[:1]     ,__UpperCamelCase		)







		def    UpperCAmelCase       (      self		:    List[Any]		)   ->						List[Any]:






				"""simple docstring"""



				lowercase__  :       Dict												=     self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__UpperCamelCase		)







		def    UpperCAmelCase       (      self		:    List[Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				lowercase__  :       Optional[int]												=     self.model_tester.prepare_config_and_inputs_for_common()
				lowercase__  :       Dict												=     True

				# in YOLOS, the seq_len is different
				lowercase__  :       List[Any]												=     self.model_tester.expected_seq_len
				for model_class in self.all_model_classes:
						lowercase__  :       int												=     True
						lowercase__  :       Any												=     False
						lowercase__  :       Tuple												=     True
						lowercase__  :       int												=     model_class(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						with torch.no_grad():
								lowercase__  :       Tuple												=     model(**self._prepare_for_class(__UpperCamelCase     ,__UpperCamelCase		)		)
						lowercase__  :       str												=     outputs.attentions
						self.assertEqual(len(__UpperCamelCase		)     ,self.model_tester.num_hidden_layers		)

						# check that output_attentions also work using config
						del inputs_dict["output_attentions"]
						lowercase__  :       List[Any]												=     True
						lowercase__  :       List[Any]												=     model_class(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						with torch.no_grad():
								lowercase__  :       Dict												=     model(**self._prepare_for_class(__UpperCamelCase     ,__UpperCamelCase		)		)
						lowercase__  :       Optional[int]												=     outputs.attentions
						self.assertEqual(len(__UpperCamelCase		)     ,self.model_tester.num_hidden_layers		)

						self.assertListEqual(
						    list(attentions[0].shape[-3:]		)     ,[self.model_tester.num_attention_heads, seq_len, seq_len]     ,)
						lowercase__  :       Union[str, Any]												=     len(__UpperCamelCase		)

						# Check attention is always last and order is fine
						lowercase__  :       int												=     True
						lowercase__  :       Any												=     True
						lowercase__  :       int												=     model_class(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						with torch.no_grad():
								lowercase__  :       List[str]												=     model(**self._prepare_for_class(__UpperCamelCase     ,__UpperCamelCase		)		)

						lowercase__  :       int												=     1
						self.assertEqual(out_len + added_hidden_states     ,len(__UpperCamelCase		)		)

						lowercase__  :       Union[str, Any]												=     outputs.attentions

						self.assertEqual(len(__UpperCamelCase		)     ,self.model_tester.num_hidden_layers		)
						self.assertListEqual(
						    list(self_attentions[0].shape[-3:]		)     ,[self.model_tester.num_attention_heads, seq_len, seq_len]     ,)







		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						List[Any]:






				"""simple docstring"""



				def check_hidden_states_output(_snake_case		:    List[str]     ,_snake_case		:    Union[str, Any]     ,_snake_case		:    List[Any]		):
						lowercase__  :       List[Any]												=     model_class(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()

						with torch.no_grad():
								lowercase__  :       Tuple												=     model(**self._prepare_for_class(__UpperCamelCase     ,__UpperCamelCase		)		)

						lowercase__  :       Any												=     outputs.hidden_states

						lowercase__  :       Any												=     getattr(
						    self.model_tester     ,'''expected_num_hidden_layers'''     ,self.model_tester.num_hidden_layers + 1		)
						self.assertEqual(len(__UpperCamelCase		)     ,__UpperCamelCase		)

						# YOLOS has a different seq_length
						lowercase__  :       List[Any]												=     self.model_tester.expected_seq_len

						self.assertListEqual(
						    list(hidden_states[0].shape[-2:]		)     ,[seq_length, self.model_tester.hidden_size]     ,)

				lowercase__  :       Union[str, Any]												=     self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
						lowercase__  :       Optional[int]												=     True
						check_hidden_states_output(__UpperCamelCase     ,__UpperCamelCase     ,__UpperCamelCase		)

						# check that output_hidden_states also work using config
						del inputs_dict["output_hidden_states"]
						lowercase__  :       str												=     True

						check_hidden_states_output(__UpperCamelCase     ,__UpperCamelCase     ,__UpperCamelCase		)







		def    UpperCAmelCase       (      self		:    Dict		)   ->						Optional[int]:






				"""simple docstring"""



				lowercase__  :       Union[str, Any]												=     self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_object_detection(*__UpperCamelCase		)







		@slow
		def    UpperCAmelCase       (      self		:    Optional[int]		)   ->						Union[str, Any]:






				"""simple docstring"""



				for model_name in YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						lowercase__  :       List[str]												=     YolosModel.from_pretrained(__UpperCamelCase		)
						self.assertIsNotNone(__UpperCamelCase		)





def 	__UpperCAmelCase (       )				-> Any:
		lowercase__  :       Union[str, Any]												=     Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''      )
		return image





@require_torch
@require_vision
class 			__A			(      unittest.TestCase       ):







		'''simple docstring'''





		@cached_property
		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						Optional[Any]:






				"""simple docstring"""



				return AutoImageProcessor.from_pretrained('''hustvl/yolos-small'''		) if is_vision_available() else None







		@slow
		def    UpperCAmelCase       (      self		:    Optional[Any]		)   ->						Tuple:






				"""simple docstring"""



				lowercase__  :       Optional[int]												=     YolosForObjectDetection.from_pretrained('''hustvl/yolos-small'''		).to(__UpperCamelCase		)

				lowercase__  :       Any												=     self.default_image_processor
				lowercase__  :       List[str]												=     prepare_img()
				lowercase__  :       Dict												=     image_processor(images=__UpperCamelCase     ,return_tensors='''pt'''		).to(__UpperCamelCase		)

				# forward pass
				with torch.no_grad():
						lowercase__  :       Union[str, Any]												=     model(inputs.pixel_values		)

				# verify outputs
				lowercase__  :       Union[str, Any]												=     torch.Size((1, 100, 92)		)
				self.assertEqual(outputs.logits.shape     ,__UpperCamelCase		)

				lowercase__  :       str												=     torch.tensor(
				    [[-24.0248, -10.3024, -14.8290], [-42.0392, -16.8200, -27.4334], [-27.2743, -11.8154, -18.7148]]     ,device=__UpperCamelCase     ,)
				lowercase__  :       Any												=     torch.tensor(
				    [[0.2559, 0.5455, 0.4706], [0.2989, 0.7279, 0.1875], [0.7732, 0.4017, 0.4462]]     ,device=__UpperCamelCase		)
				self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]     ,__UpperCamelCase     ,atol=1e-4		)		)
				self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3]     ,__UpperCamelCase     ,atol=1e-4		)		)

				# verify postprocessing
				lowercase__  :       Optional[int]												=     image_processor.post_process_object_detection(
				    __UpperCamelCase     ,threshold=0.3     ,target_sizes=[image.size[::-1]]		)[0]
				lowercase__  :       List[str]												=     torch.tensor([0.9994, 0.9790, 0.9964, 0.9972, 0.9861]		).to(__UpperCamelCase		)
				lowercase__  :       List[Any]												=     [75, 75, 17, 63, 17]
				lowercase__  :       Optional[Any]												=     torch.tensor([335.0609, 79.3848, 375.4216, 187.2495]		).to(__UpperCamelCase		)

				self.assertEqual(len(results['''scores''']		)     ,5		)
				self.assertTrue(torch.allclose(results['''scores''']     ,__UpperCamelCase     ,atol=1e-4		)		)
				self.assertSequenceEqual(results['''labels'''].tolist()     ,__UpperCamelCase		)
				self.assertTrue(torch.allclose(results['''boxes'''][0, :]     ,__UpperCamelCase		)		)

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE		:     Dict         =	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

SCREAMING_SNAKE_CASE		:     int         =	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
SCREAMING_SNAKE_CASE		:     Any         =	{F'''funnel-transformer/{name}''': 512 for name in _model_names}
SCREAMING_SNAKE_CASE		:     Tuple         =	{F'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class        UpperCamelCase						(				__a				):
						a__		:Dict      =			VOCAB_FILES_NAMES
						a__		:Dict      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Dict      =			PRETRAINED_INIT_CONFIGURATION
						a__		:str      =			FunnelTokenizer
						a__		:List[str]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:int      =			2





						def __init__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=True			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase="<sep>"			,     __UpperCamelCase="<pad>"			,     __UpperCamelCase="<cls>"			,     __UpperCamelCase="<mask>"			,     __UpperCamelCase="<s>"			,     __UpperCamelCase="</s>"			,     __UpperCamelCase=True			,     __UpperCamelCase=True			,     __UpperCamelCase=None			,     __UpperCamelCase="##"			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(
												    __UpperCamelCase			,     tokenizer_file=__UpperCamelCase			,     do_lower_case=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     sep_token=__UpperCamelCase			,     pad_token=__UpperCamelCase			,     cls_token=__UpperCamelCase			,     mask_token=__UpperCamelCase			,     bos_token=__UpperCamelCase			,     eos_token=__UpperCamelCase			,     clean_text=__UpperCamelCase			,     tokenize_chinese_chars=__UpperCamelCase			,     strip_accents=__UpperCamelCase			,     wordpieces_prefix=__UpperCamelCase			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
												if (
												    normalizer_state.get("""lowercase"""			,     __UpperCamelCase						) != do_lower_case
												    or normalizer_state.get("""strip_accents"""			,     __UpperCamelCase						) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""			,     __UpperCamelCase						) != tokenize_chinese_chars
												):
																		UpperCamelCase_		:							List[Any]  =		getattr(__UpperCamelCase			,     normalizer_state.pop("""type"""						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		do_lower_case
																		UpperCamelCase_		:							Tuple  =		strip_accents
																		UpperCamelCase_		:							Dict  =		tokenize_chinese_chars
																		UpperCamelCase_		:							Union[str, Any]  =		normalizer_class(**__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		do_lower_case





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None						)				->				Dict:
												UpperCamelCase_		:							Any  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				List[int]:
												UpperCamelCase_		:							Optional[Any]  =		[self.sep_token_id]
												UpperCamelCase_		:							Any  =		[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
												return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												UpperCamelCase_		:							int  =		self._tokenizer.model.save(__UpperCamelCase			,     name=__UpperCamelCase						)
												return tuple(__UpperCamelCase						)






