'''simple docstring'''

import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

      if is_vision_available():
            from transformers import MaskFormerImageProcessor

if is_vision_available():
      from PIL import Image
class 				lowercase_     :







    def __init__(						self							,  UpperCamelCase__							,  UpperCamelCase__=2							,  UpperCamelCase__=True							,  UpperCamelCase__=False							,  UpperCamelCase__=1_0							,  UpperCamelCase__=3							,  UpperCamelCase__=3_2 * 4							,  UpperCamelCase__=3_2 * 6							,  UpperCamelCase__=4							,  UpperCamelCase__=3_2							,  )					->							Tuple:





     """simple docstring"""

     UpperCAmelCase_												=       parent
     UpperCAmelCase_												=       batch_size
     UpperCAmelCase_												=       is_training
     UpperCAmelCase_												=       use_auxiliary_loss
     UpperCAmelCase_												=       num_queries
     UpperCAmelCase_												=       num_channels
     UpperCAmelCase_												=       min_size
     UpperCAmelCase_												=       max_size
     UpperCAmelCase_												=       num_labels
     UpperCAmelCase_												=       mask_feature_size







    def 	lowerCamelCase_     (						self					)					->							Tuple:





     """simple docstring"""

     UpperCAmelCase_												=       floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]					).to(
         __lowerCAmelCase					)

     UpperCAmelCase_												=       torch.ones([self.batch_size, self.min_size, self.max_size]							,  device=__lowerCAmelCase					)

     UpperCAmelCase_												=       (
         torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]							,  device=__lowerCAmelCase					) > 0.5
     ).float()
     UpperCAmelCase_												=       (torch.rand((self.batch_size, self.num_labels)							,  device=__lowerCAmelCase					) > 0.5).long()

     UpperCAmelCase_												=       self.get_config()
     return config, pixel_values, pixel_mask, mask_labels, class_labels







    def 	lowerCamelCase_     (						self					)					->							Optional[int]:





     """simple docstring"""

     return MaskFormerConfig.from_backbone_and_decoder_configs(
         backbone_config=SwinConfig(
             depths=[1, 1, 1, 1]							,  )							,  decoder_config=DetrConfig(
             decoder_ffn_dim=1_2_8							,  num_queries=self.num_queries							,  decoder_attention_heads=2							,  d_model=self.mask_feature_size							,  )							,  mask_feature_size=self.mask_feature_size							,  fpn_feature_size=self.mask_feature_size							,  num_channels=self.num_channels							,  num_labels=self.num_labels							,  )







    def 	lowerCamelCase_     (						self					)					->							Union[str, Any]:





     """simple docstring"""

     UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_												=       self.prepare_config_and_inputs()
     UpperCAmelCase_												=       {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
     return config, inputs_dict







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__					)					->							int:





     """simple docstring"""

     UpperCAmelCase_												=       output.encoder_hidden_states
     UpperCAmelCase_												=       output.pixel_decoder_hidden_states
     UpperCAmelCase_												=       output.transformer_decoder_hidden_states

     self.parent.assertTrue(len(__lowerCAmelCase					)							,  len(config.backbone_config.depths					)					)
     self.parent.assertTrue(len(__lowerCAmelCase					)							,  len(config.backbone_config.depths					)					)
     self.parent.assertTrue(len(__lowerCAmelCase					)							,  config.decoder_config.decoder_layers					)







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__=False					)					->							Union[str, Any]:





     """simple docstring"""

     with torch.no_grad():
      UpperCAmelCase_												=       MaskFormerModel(config=__lowerCAmelCase					)
      model.to(__lowerCAmelCase					)
      model.eval()

      UpperCAmelCase_												=       model(pixel_values=__lowerCAmelCase							,  pixel_mask=__lowerCAmelCase					)
      UpperCAmelCase_												=       model(__lowerCAmelCase							,  output_hidden_states=__lowerCAmelCase					)
     # the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
     # encoder and pixel decoder
     self.parent.assertEqual(
         output.transformer_decoder_last_hidden_state.shape							,  (self.batch_size, self.num_queries, self.mask_feature_size)							,  )
     # let's ensure the other two hidden state exists
     self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None					)
     self.parent.assertTrue(output.encoder_last_hidden_state is not None					)

     if output_hidden_states:
      self.check_output_hidden_state(__lowerCAmelCase							,  __lowerCAmelCase					)







    def 	lowerCamelCase_     (						self							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__							,  UpperCamelCase__					)					->							Optional[Any]:





     """simple docstring"""

     UpperCAmelCase_												=       MaskFormerForInstanceSegmentation(config=__lowerCAmelCase					)
     model.to(__lowerCAmelCase					)
     model.eval()

     def comm_check_on_output(UpperCamelCase__					):
      # let's still check that all the required stuff is there
      self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None					)
      self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None					)
      self.parent.assertTrue(result.encoder_last_hidden_state is not None					)
      # okay, now we need to check the logits shape
      # due to the encoder compression, masks have a //4 spatial size
      self.parent.assertEqual(
          result.masks_queries_logits.shape							,  (self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)							,  )
      # + 1 for null class
      self.parent.assertEqual(
          result.class_queries_logits.shape							,  (self.batch_size, self.num_queries, self.num_labels + 1)					)

     with torch.no_grad():
      UpperCAmelCase_												=       model(pixel_values=__lowerCAmelCase							,  pixel_mask=__lowerCAmelCase					)
      UpperCAmelCase_												=       model(__lowerCAmelCase					)

      comm_check_on_output(__lowerCAmelCase					)

      UpperCAmelCase_												=       model(
          pixel_values=__lowerCAmelCase							,  pixel_mask=__lowerCAmelCase							,  mask_labels=__lowerCAmelCase							,  class_labels=__lowerCAmelCase					)

     comm_check_on_output(__lowerCAmelCase					)

     self.parent.assertTrue(result.loss is not None					)
     self.parent.assertEqual(result.loss.shape							,  torch.Size([1]					)					)


@require_torch
class 				lowercase_     (  _a			,	_a			,	unittest.TestCase    ):
    a_   							= (MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
    a_   							= (
        {"""feature-extraction""": MaskFormerModel, """image-segmentation""": MaskFormerForInstanceSegmentation}
        if is_torch_available()
        else {}
    )

    a_   							= False
    a_   							= False
    a_   							= False
    a_   							= False







    def 	lowerCamelCase_     (						self					)					->							Union[str, Any]:





     """simple docstring"""

     UpperCAmelCase_												=       MaskFormerModelTester(self					)
     UpperCAmelCase_												=       ConfigTester(self							,  config_class=__lowerCAmelCase							,  has_text_modality=__lowerCAmelCase					)







    def 	lowerCamelCase_     (						self					)					->							List[Any]:





     """simple docstring"""

     self.config_tester.run_common_tests()







    def 	lowerCamelCase_     (						self					)					->							Optional[Any]:





     """simple docstring"""

     UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.create_and_check_maskformer_model(__lowerCAmelCase							,  **__lowerCAmelCase							,  output_hidden_states=__lowerCAmelCase					)







    def 	lowerCamelCase_     (						self					)					->							Optional[int]:





     """simple docstring"""

     UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*__lowerCAmelCase					)







    @unittest.skip(reason="MaskFormer does not use inputs_embeds"					)
    def 	lowerCamelCase_     (						self					)					->							List[str]:





     """simple docstring"""

     pass







    @unittest.skip(reason="MaskFormer does not have a get_input_embeddings method"					)
    def 	lowerCamelCase_     (						self					)					->							Dict:





     """simple docstring"""

     pass







    @unittest.skip(reason="MaskFormer is not a generative model"					)
    def 	lowerCamelCase_     (						self					)					->							str:





     """simple docstring"""

     pass







    @unittest.skip(reason="MaskFormer does not use token embeddings"					)
    def 	lowerCamelCase_     (						self					)					->							Dict:





     """simple docstring"""

     pass







    @require_torch_multi_gpu
    @unittest.skip(
        reason="MaskFormer has some layers using `add_module` which doesn't work well with `nn.DataParallel`"					)
    def 	lowerCamelCase_     (						self					)					->							Union[str, Any]:





     """simple docstring"""

     pass







    @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."					)
    def 	lowerCamelCase_     (						self					)					->							List[str]:





     """simple docstring"""

     pass







    def 	lowerCamelCase_     (						self					)					->							Optional[int]:





     """simple docstring"""

     UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      UpperCAmelCase_												=       model_class(__lowerCAmelCase					)
      UpperCAmelCase_												=       inspect.signature(model.forward					)
      # signature.parameters is an OrderedDict => so arg_names order is deterministic
      UpperCAmelCase_												=       [*signature.parameters.keys()]

      UpperCAmelCase_												=       ["pixel_values"]
      self.assertListEqual(arg_names[:1]							,  __lowerCAmelCase					)







    @slow
    def 	lowerCamelCase_     (						self					)					->							Tuple:





     """simple docstring"""

     for model_name in ["facebook/maskformer-swin-small-coco"]:
      UpperCAmelCase_												=       MaskFormerModel.from_pretrained(__lowerCAmelCase					)
      self.assertIsNotNone(__lowerCAmelCase					)







    def 	lowerCamelCase_     (						self					)					->							Optional[Any]:





     """simple docstring"""

     UpperCAmelCase_												=       (self.model_tester.min_size,) * 2
     UpperCAmelCase_												=       {
         "pixel_values": torch.randn((2, 3, *size)							,  device=__lowerCAmelCase					),
         "mask_labels": torch.randn((2, 1_0, *size)							,  device=__lowerCAmelCase					),
         "class_labels": torch.zeros(2							,  1_0							,  device=__lowerCAmelCase					).long(),
     }

     UpperCAmelCase_												=       MaskFormerForInstanceSegmentation(MaskFormerConfig()					).to(__lowerCAmelCase					)
     UpperCAmelCase_												=       model(**__lowerCAmelCase					)
     self.assertTrue(outputs.loss is not None					)







    def 	lowerCamelCase_     (						self					)					->							Any:





     """simple docstring"""

     UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.create_and_check_maskformer_model(__lowerCAmelCase							,  **__lowerCAmelCase							,  output_hidden_states=__lowerCAmelCase					)







    def 	lowerCamelCase_     (						self					)					->							Dict:





     """simple docstring"""

     UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs_for_common()

     for model_class in self.all_model_classes:
      UpperCAmelCase_												=       model_class(__lowerCAmelCase					).to(__lowerCAmelCase					)
      UpperCAmelCase_												=       model(**__lowerCAmelCase							,  output_attentions=__lowerCAmelCase					)
      self.assertTrue(outputs.attentions is not None					)







    def 	lowerCamelCase_     (						self					)					->							str:





     """simple docstring"""

     if not self.model_tester.is_training:
      return
     # only MaskFormerForInstanceSegmentation has the loss
     UpperCAmelCase_												=       self.all_model_classes[1]
     UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs()

     UpperCAmelCase_												=       model_class(__lowerCAmelCase					)
     model.to(__lowerCAmelCase					)
     model.train()

     UpperCAmelCase_												=       model(__lowerCAmelCase							,  mask_labels=__lowerCAmelCase							,  class_labels=__lowerCAmelCase					).loss
     loss.backward()







    def 	lowerCamelCase_     (						self					)					->							Tuple:





     """simple docstring"""

     UpperCAmelCase_												=       self.all_model_classes[1]
     UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_							,							UpperCAmelCase_												=       self.model_tester.prepare_config_and_inputs()
     UpperCAmelCase_												=       True
     UpperCAmelCase_												=       True

     UpperCAmelCase_												=       model_class(__lowerCAmelCase					)
     model.to(__lowerCAmelCase					)
     model.train()

     UpperCAmelCase_												=       model(__lowerCAmelCase							,  mask_labels=__lowerCAmelCase							,  class_labels=__lowerCAmelCase					)

     UpperCAmelCase_												=       outputs.encoder_hidden_states[0]
     encoder_hidden_states.retain_grad()

     UpperCAmelCase_												=       outputs.pixel_decoder_hidden_states[0]
     pixel_decoder_hidden_states.retain_grad()
     # we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
     UpperCAmelCase_												=       outputs.transformer_decoder_hidden_states[0]
     transformer_decoder_hidden_states.retain_grad()

     UpperCAmelCase_												=       outputs.attentions[0]
     attentions.retain_grad()

     outputs.loss.backward(retain_graph=__lowerCAmelCase					)

     self.assertIsNotNone(encoder_hidden_states.grad					)
     self.assertIsNotNone(pixel_decoder_hidden_states.grad					)
     self.assertIsNotNone(transformer_decoder_hidden_states.grad					)
     self.assertIsNotNone(attentions.grad					)


__snake_case							: str			=     1e-4



def lowerCamelCase__    (							):
 UpperCAmelCase_												=       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"							)
 return image


@require_vision
@slow
class 				lowercase_     (  unittest.TestCase    ):







    @cached_property
    def 	lowerCamelCase_     (						self					)					->							Tuple:





     """simple docstring"""

     return (
         MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-small-coco"					)
         if is_vision_available()
         else None
     )







    def 	lowerCamelCase_     (						self					)					->							Union[str, Any]:





     """simple docstring"""

     UpperCAmelCase_												=       MaskFormerModel.from_pretrained("facebook/maskformer-swin-small-coco"					).to(__lowerCAmelCase					)
     UpperCAmelCase_												=       self.default_image_processor
     UpperCAmelCase_												=       prepare_img()
     UpperCAmelCase_												=       image_processor(__lowerCAmelCase							,  return_tensors="pt"					).to(__lowerCAmelCase					)
     UpperCAmelCase_												=       inputs["pixel_values"].shape
     # check size is divisible by 32
     self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0					)
     # check size
     self.assertEqual(__lowerCAmelCase							,  (1, 3, 8_0_0, 1_0_8_8)					)

     with torch.no_grad():
      UpperCAmelCase_												=       model(**__lowerCAmelCase					)

     UpperCAmelCase_												=       torch.tensor(
         [[-0.0482, 0.9228, 0.4951], [-0.2547, 0.8017, 0.8527], [-0.0069, 0.3385, -0.0089]]					).to(__lowerCAmelCase					)
     self.assertTrue(
         torch.allclose(
             outputs.encoder_last_hidden_state[0, 0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)

     UpperCAmelCase_												=       torch.tensor(
         [[-0.8422, -0.8434, -0.9718], [-1.0144, -0.5565, -0.4195], [-1.0038, -0.4484, -0.1961]]					).to(__lowerCAmelCase					)
     self.assertTrue(
         torch.allclose(
             outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)

     UpperCAmelCase_												=       torch.tensor(
         [[0.2852, -0.0159, 0.9735], [0.6254, 0.1858, 0.8529], [-0.0680, -0.4116, 1.8413]]					).to(__lowerCAmelCase					)
     self.assertTrue(
         torch.allclose(
             outputs.transformer_decoder_last_hidden_state[0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)







    def 	lowerCamelCase_     (						self					)					->							List[Any]:





     """simple docstring"""

     UpperCAmelCase_												=       (
         MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"					)
         .to(__lowerCAmelCase					)
         .eval()
     )
     UpperCAmelCase_												=       self.default_image_processor
     UpperCAmelCase_												=       prepare_img()
     UpperCAmelCase_												=       image_processor(__lowerCAmelCase							,  return_tensors="pt"					).to(__lowerCAmelCase					)
     UpperCAmelCase_												=       inputs["pixel_values"].shape
     # check size is divisible by 32
     self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0					)
     # check size
     self.assertEqual(__lowerCAmelCase							,  (1, 3, 8_0_0, 1_0_8_8)					)

     with torch.no_grad():
      UpperCAmelCase_												=       model(**__lowerCAmelCase					)
     # masks_queries_logits
     UpperCAmelCase_												=       outputs.masks_queries_logits
     self.assertEqual(
         masks_queries_logits.shape							,  (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)							,  )
     UpperCAmelCase_												=       [
         [-1.3737124, -1.7724937, -1.9364233],
         [-1.5977281, -1.9867939, -2.1523695],
         [-1.5795398, -1.9269832, -2.093942],
     ]
     UpperCAmelCase_												=       torch.tensor(__lowerCAmelCase					).to(__lowerCAmelCase					)
     self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)
     # class_queries_logits
     UpperCAmelCase_												=       outputs.class_queries_logits
     self.assertEqual(
         class_queries_logits.shape							,  (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)					)
     UpperCAmelCase_												=       torch.tensor(
         [
             [1.65_12e00, -5.25_72e00, -3.35_19e00],
             [3.61_69e-02, -5.90_25e00, -2.93_13e00],
             [1.07_66e-04, -7.76_30e00, -5.12_63e00],
         ]					).to(__lowerCAmelCase					)
     self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)







    def 	lowerCamelCase_     (						self					)					->							Union[str, Any]:





     """simple docstring"""

     UpperCAmelCase_												=       (
         MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-resnet101-coco-stuff"					)
         .to(__lowerCAmelCase					)
         .eval()
     )
     UpperCAmelCase_												=       self.default_image_processor
     UpperCAmelCase_												=       prepare_img()
     UpperCAmelCase_												=       image_processor(__lowerCAmelCase							,  return_tensors="pt"					).to(__lowerCAmelCase					)
     UpperCAmelCase_												=       inputs["pixel_values"].shape
     # check size is divisible by 32
     self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0					)
     # check size
     self.assertEqual(__lowerCAmelCase							,  (1, 3, 8_0_0, 1_0_8_8)					)

     with torch.no_grad():
      UpperCAmelCase_												=       model(**__lowerCAmelCase					)
     # masks_queries_logits
     UpperCAmelCase_												=       outputs.masks_queries_logits
     self.assertEqual(
         masks_queries_logits.shape							,  (1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)							,  )
     UpperCAmelCase_												=       [[-0.9046, -2.6366, -4.6062], [-3.4179, -5.7890, -8.8057], [-4.9179, -7.6560, -10.7711]]
     UpperCAmelCase_												=       torch.tensor(__lowerCAmelCase					).to(__lowerCAmelCase					)
     self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)
     # class_queries_logits
     UpperCAmelCase_												=       outputs.class_queries_logits
     self.assertEqual(
         class_queries_logits.shape							,  (1, model.config.decoder_config.num_queries, model.config.num_labels + 1)					)
     UpperCAmelCase_												=       torch.tensor(
         [[4.7188, -3.2585, -2.8857], [6.6871, -2.9181, -1.2487], [7.2449, -2.2764, -2.1874]]					).to(__lowerCAmelCase					)
     self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]							,  __lowerCAmelCase							,  atol=__lowerCAmelCase					)					)







    def 	lowerCamelCase_     (						self					)					->							List[Any]:





     """simple docstring"""

     UpperCAmelCase_												=       (
         MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"					)
         .to(__lowerCAmelCase					)
         .eval()
     )
     UpperCAmelCase_												=       self.default_image_processor

     UpperCAmelCase_												=       image_processor(
         [np.zeros((3, 8_0_0, 1_3_3_3)					), np.zeros((3, 8_0_0, 1_3_3_3)					)]							,  segmentation_maps=[np.zeros((3_8_4, 3_8_4)					).astype(np.floataa					), np.zeros((3_8_4, 3_8_4)					).astype(np.floataa					)]							,  return_tensors="pt"							,  )

     UpperCAmelCase_												=       inputs["pixel_values"].to(__lowerCAmelCase					)
     UpperCAmelCase_												=       [el.to(__lowerCAmelCase					) for el in inputs["mask_labels"]]
     UpperCAmelCase_												=       [el.to(__lowerCAmelCase					) for el in inputs["class_labels"]]

     with torch.no_grad():
      UpperCAmelCase_												=       model(**__lowerCAmelCase					)

     self.assertTrue(outputs.loss is not None					)







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCamelCase__           =			logging.get_logger(__name__)

UpperCamelCase__           =			{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__           =			{
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

UpperCamelCase__           =			{
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							VOCAB_FILES_NAMES
							snake_case  :   Optional[Any]      =							PRETRAINED_VOCAB_FILES_MAP
							snake_case  :   Any      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case  :   Dict      =							["""input_ids""", """attention_mask"""]
							snake_case  :   List[Any]      =							RobertaTokenizer






							def __init__(    self	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase=None	,			__lowerCAmelCase="replace"	,			__lowerCAmelCase="<s>"	,			__lowerCAmelCase="</s>"	,			__lowerCAmelCase="</s>"	,			__lowerCAmelCase="<s>"	,			__lowerCAmelCase="<unk>"	,			__lowerCAmelCase="<pad>"	,			__lowerCAmelCase="<mask>"	,			__lowerCAmelCase=False	,			__lowerCAmelCase=True	,			**__lowerCAmelCase	,			):
														super().__init__(
														    __lowerCAmelCase	,			__lowerCAmelCase	,			tokenizer_file=__lowerCAmelCase	,			errors=__lowerCAmelCase	,			bos_token=__lowerCAmelCase	,			eos_token=__lowerCAmelCase	,			sep_token=__lowerCAmelCase	,			cls_token=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			mask_token=__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	,			trim_offsets=__lowerCAmelCase	,			**__lowerCAmelCase	,			)

														UpperCamelCase__					  =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
														if pre_tok_state.get("""add_prefix_space"""	,			__lowerCAmelCase	) != add_prefix_space:
																					UpperCamelCase__					  =      getattr(__lowerCAmelCase	,			pre_tok_state.pop("""type"""	)	)
																					UpperCamelCase__					  =      add_prefix_space
																					UpperCamelCase__					  =      pre_tok_class(**__lowerCAmelCase	)

														UpperCamelCase__					  =      add_prefix_space

														UpperCamelCase__					  =      """post_processor"""
														UpperCamelCase__					  =      getattr(self.backend_tokenizer	,			__lowerCAmelCase	,			__lowerCAmelCase	)
														if tokenizer_component_instance:
																					UpperCamelCase__					  =      json.loads(tokenizer_component_instance.__getstate__()	)

																					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																					if "sep" in state:
																												UpperCamelCase__					  =      tuple(state["""sep"""]	)
																					if "cls" in state:
																												UpperCamelCase__					  =      tuple(state["""cls"""]	)

																					UpperCamelCase__					  =      False

																					if state.get("""add_prefix_space"""	,			__lowerCAmelCase	) != add_prefix_space:
																												UpperCamelCase__					  =      add_prefix_space
																												UpperCamelCase__					  =      True

																					if state.get("""trim_offsets"""	,			__lowerCAmelCase	) != trim_offsets:
																												UpperCamelCase__					  =      trim_offsets
																												UpperCamelCase__					  =      True

																					if changes_to_apply:
																												UpperCamelCase__					  =      getattr(__lowerCAmelCase	,			state.pop("""type"""	)	)
																												UpperCamelCase__					  =      component_class(**__lowerCAmelCase	)
																												setattr(self.backend_tokenizer	,			__lowerCAmelCase	,			__lowerCAmelCase	)






							@property
							def 					_lowerCamelCase		(    self	):
														if self._mask_token is None:
																					if self.verbose:
																												logger.error("""Using mask_token, but it is not set yet."""	)
																					return None
														return str(self._mask_token	)






							@mask_token.setter
							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	) if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	) else value
														UpperCamelCase__					  =      value






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      kwargs.get("""is_split_into_words"""	,			__lowerCAmelCase	)
														assert self.add_prefix_space or not is_split_into_words, (
														    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*__lowerCAmelCase	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														UpperCamelCase__					  =      kwargs.get("""is_split_into_words"""	,			__lowerCAmelCase	)

														assert self.add_prefix_space or not is_split_into_words, (
														    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*__lowerCAmelCase	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      self._tokenizer.model.save(__lowerCAmelCase	,			name=__lowerCAmelCase	)
														return tuple(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=None	):
														UpperCamelCase__					  =      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
														if token_ids_a is None:
																					return output

														return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep	) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]




