






'''simple docstring'''
import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase				):


   """simple docstring"""




   lowercase_						:     Optional[int]							     =  [2, 2, 6, 2] if "tiny" in model_name else [2, 2, 18, 2]
   lowercase_						:     Dict							     =  True if "large" in model_name or "huge" in model_name else False
   lowercase_						:     Any							     =  True if "large" in model_name or "huge" in model_name else False
   lowercase_						:     List[str]							     =  True if "large" in model_name or "huge" in model_name else False

   if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
      if "fl3" in model_name:
         lowercase_						:     Tuple							     =  [3, 3, 3, 3]
         lowercase_						:     str							     =  [5, 5, 5, 5]
      elif "fl4" in model_name:
         lowercase_						:     List[Any]							     =  [4, 4, 4, 4]
         lowercase_						:     Union[str, Any]							     =  [3, 3, 3, 3]

   if "tiny" in model_name or "small" in model_name or "base" in model_name:
      lowercase_						:     List[str]							     =  [3, 3, 3, 3]
      if "lrf" in model_name:
         lowercase_						:     List[str]							     =  [3, 3, 3, 3]
      else:
         lowercase_						:     int							     =  [2, 2, 2, 2]

   if "tiny" in model_name:
      lowercase_						:     Dict							     =  96
   elif "small" in model_name:
      lowercase_						:     int							     =  96
   elif "base" in model_name:
      lowercase_						:     Dict							     =  128
   elif "large" in model_name:
      lowercase_						:     Union[str, Any]							     =  192
   elif "xlarge" in model_name:
      lowercase_						:     List[Any]							     =  256
   elif "huge" in model_name:
      lowercase_						:     Optional[Any]							     =  352

   # set label information
   lowercase_						:     Optional[Any]							     =  "huggingface/label-files"
   if "large" in model_name or "huge" in model_name:
      lowercase_						:     Dict							     =  "imagenet-22k-id2label.json"
   else:
      lowercase_						:     Union[str, Any]							     =  "imagenet-1k-id2label.json"

   lowercase_						:     Optional[Any]							     =  json.load(open(hf_hub_download(_UpperCamelCase    ,		_UpperCamelCase    ,		repo_type="dataset"				)    ,		"r"				)				)
   lowercase_						:     Any							     =  {int(_UpperCamelCase				): v for k, v in idalabel.items()}
   lowercase_						:     Optional[int]							     =  {v: k for k, v in idalabel.items()}

   lowercase_						:     List[str]							     =  FocalNetConfig(
       embed_dim=_UpperCamelCase    ,		depths=_UpperCamelCase    ,		focal_levels=_UpperCamelCase    ,		focal_windows=_UpperCamelCase    ,		use_conv_embed=_UpperCamelCase    ,		idalabel=_UpperCamelCase    ,		labelaid=_UpperCamelCase    ,		use_post_layernorm=_UpperCamelCase    ,		use_layerscale=_UpperCamelCase    ,		)

   return config




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase				):


   """simple docstring"""




   if "patch_embed.proj" in name:
      lowercase_						:     int							     =  name.replace("patch_embed.proj"    ,		"embeddings.patch_embeddings.projection"				)
   if "patch_embed.norm" in name:
      lowercase_						:     Union[str, Any]							     =  name.replace("patch_embed.norm"    ,		"embeddings.norm"				)
   if "layers" in name:
      lowercase_						:     Optional[Any]							     =  "encoder." + name
   if "encoder.layers" in name:
      lowercase_						:     int							     =  name.replace("encoder.layers"    ,		"encoder.stages"				)
   if "downsample.proj" in name:
      lowercase_						:     Dict							     =  name.replace("downsample.proj"    ,		"downsample.projection"				)
   if "blocks" in name:
      lowercase_						:     Dict							     =  name.replace("blocks"    ,		"layers"				)
   if "modulation.f.weight" in name or "modulation.f.bias" in name:
      lowercase_						:     Optional[Any]							     =  name.replace("modulation.f"    ,		"modulation.projection_in"				)
   if "modulation.h.weight" in name or "modulation.h.bias" in name:
      lowercase_						:     List[str]							     =  name.replace("modulation.h"    ,		"modulation.projection_context"				)
   if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
      lowercase_						:     Optional[Any]							     =  name.replace("modulation.proj"    ,		"modulation.projection_out"				)

   if name == "norm.weight":
      lowercase_						:     Optional[int]							     =  "layernorm.weight"
   if name == "norm.bias":
      lowercase_						:     Dict							     =  "layernorm.bias"

   if "head" in name:
      lowercase_						:     Optional[int]							     =  name.replace("head"    ,		"classifier"				)
   else:
      lowercase_						:     List[Any]							     =  "focalnet." + name

   return name




def 		__SCREAMING_SNAKE_CASE       (	_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=False				):


   """simple docstring"""




   lowercase_						:     Optional[Any]							     =  {
       "focalnet-tiny": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth",
       "focalnet-tiny-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth",
       "focalnet-small": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth",
       "focalnet-small-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth",
       "focalnet-base": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth",
       "focalnet-base-lrf": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth",
       "focalnet-large-lrf-fl3": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth",
       "focalnet-large-lrf-fl4": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth",
       "focalnet-xlarge-lrf-fl3": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth",
       "focalnet-xlarge-lrf-fl4": "https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth",
   }
   # fmt: on

   lowercase_						:     List[Any]							     =  model_name_to_url[model_name]
   print("Checkpoint URL: "    ,		_UpperCamelCase				)
   lowercase_						:     Optional[int]							     =  torch.hub.load_state_dict_from_url(_UpperCamelCase    ,		map_location="cpu"				)["model"]

   # rename keys
   for key in state_dict.copy().keys():
      lowercase_						:     int							     =  state_dict.pop(_UpperCamelCase				)
      lowercase_						:     Optional[int]							     =  val

   lowercase_						:     Optional[int]							     =  get_focalnet_config(_UpperCamelCase				)
   lowercase_						:     Any							     =  FocalNetForImageClassification(_UpperCamelCase				)
   model.eval()

   # load state dict
   model.load_state_dict(_UpperCamelCase				)

   # verify conversion
   lowercase_						:     List[str]							     =  "http://images.cocodataset.org/val2017/000000039769.jpg"

   lowercase_						:     int							     =  BitImageProcessor(
       do_resize=_UpperCamelCase    ,		size={"shortest_edge": 256}    ,		resample=PILImageResampling.BILINEAR    ,		do_center_crop=_UpperCamelCase    ,		crop_size=224    ,		do_normalize=_UpperCamelCase    ,		image_mean=_UpperCamelCase    ,		image_std=_UpperCamelCase    ,		)
   lowercase_						:     Dict							     =  Image.open(requests.get(_UpperCamelCase    ,		stream=_UpperCamelCase				).raw				)
   lowercase_						:     List[Any]							     =  processor(images=_UpperCamelCase    ,		return_tensors="pt"				)

   lowercase_						:     Union[str, Any]							     =  transforms.Compose(
       [
           transforms.Resize(256				),
           transforms.CenterCrop(224				),
           transforms.ToTensor(),
           transforms.Normalize(mean=[0.485, 0.456, 0.406]    ,		std=[0.229, 0.224, 0.225]				),
       ]				)

   lowercase_						:     Dict							     =  image_transforms(_UpperCamelCase				).unsqueeze(0				)

   # verify pixel_values
   assert torch.allclose(inputs.pixel_values    ,		_UpperCamelCase    ,		atol=1e-4				)

   lowercase_						:     List[str]							     =  model(**_UpperCamelCase				)

   lowercase_						:     Dict							     =  outputs.logits.argmax(-1				).item()
   print("Predicted class:"    ,		model.config.idalabel[predicted_class_idx]				)

   print("First values of logits:"    ,		outputs.logits[0, :3]				)

   if model_name == "focalnet-tiny":
      lowercase_						:     Optional[Any]							     =  torch.tensor([0.2166, -0.4368, 0.2191]				)
   elif model_name == "focalnet-tiny-lrf":
      lowercase_						:     Dict							     =  torch.tensor([1.1669, 0.0125, -0.1695]				)
   elif model_name == "focalnet-small":
      lowercase_						:     Optional[Any]							     =  torch.tensor([0.4917, -0.0430, 0.1341]				)
   elif model_name == "focalnet-small-lrf":
      lowercase_						:     Any							     =  torch.tensor([-0.2588, -0.5342, -0.2331]				)
   elif model_name == "focalnet-base":
      lowercase_						:     Dict							     =  torch.tensor([-0.1655, -0.4090, -0.1730]				)
   elif model_name == "focalnet-base-lrf":
      lowercase_						:     List[Any]							     =  torch.tensor([0.5306, -0.0483, -0.3928]				)
   assert torch.allclose(outputs.logits[0, :3]    ,		_UpperCamelCase    ,		atol=1e-4				)
   print("Looks ok!"				)

   if pytorch_dump_folder_path is not None:
      print(F"""Saving model and processor of {model_name} to {pytorch_dump_folder_path}"""				)
      model.save_pretrained(_UpperCamelCase				)
      processor.save_pretrained(_UpperCamelCase				)

   if push_to_hub:
      print(F"""Pushing model and processor of {model_name} to the hub..."""				)
      model.push_to_hub(F"""{model_name}"""				)
      processor.push_to_hub(F"""{model_name}"""				)


if __name__ == "__main__":
 UpperCamelCase__			=      argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--model_name',
     default='focalnet-tiny',
     type=str,
     help='Name of the FocalNet model you\'d like to convert.',
 )
 parser.add_argument(
     '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
 )
 parser.add_argument(
     '--push_to_hub',
     action='store_true',
     help='Whether to push the model and processor to the hub.',
 )

 UpperCamelCase__			=      parser.parse_args()
 convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)












'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 							_UpperCAmelCase    (		snake_case					,       unittest.TestCase      ):
    __lowerCamelCase: Optional[int]									=     BertTokenizer
    __lowerCamelCase: Optional[int]									=     BertTokenizerFast
    __lowerCamelCase: Union[str, Any]									=     True
    __lowerCamelCase: int									=     True
    __lowerCamelCase: Optional[Any]									=     filter_non_english




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       super().setUp()

       lowercase_						:     Optional[Any]							     =  [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase_						:     Optional[Any]							     =  os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file	,  "w"	,  encoding="utf-8"      ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )




    def 					lowerCAmelCase__					(      self	:							Any	,  a	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  "UNwant\u00E9d,running"
       lowercase_						:     List[str]							     =  "unwanted, running"
       return input_text, output_text




    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  self.tokenizer_class(self.vocab_file      )

       lowercase_						:     Union[str, Any]							     =  tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(a	,  ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a      )	,  [9, 6, 7, 1_2, 1_0, 1_1]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       if not self.test_rust_tokenizer:
          return

       lowercase_						:     Optional[Any]							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       lowercase_						:     int							     =  "UNwant\u00E9d,running"

       lowercase_						:     Dict							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     List[Any]							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     List[Any]							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Optional[int]							     =  self.get_rust_tokenizer()
       lowercase_						:     Optional[int]							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )

       # With lower casing
       lowercase_						:     Tuple							     =  self.get_tokenizer(do_lower_case=a      )
       lowercase_						:     Dict							     =  self.get_rust_tokenizer(do_lower_case=a      )

       lowercase_						:     List[Any]							     =  "UNwant\u00E9d,running"

       lowercase_						:     str							     =  tokenizer.tokenize(a      )
       lowercase_						:     Optional[Any]							     =  rust_tokenizer.tokenize(a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Any							     =  tokenizer.encode(a	,  add_special_tokens=a      )
       lowercase_						:     str							     =  rust_tokenizer.encode(a	,  add_special_tokens=a      )
       self.assertListEqual(a	,  a      )

       lowercase_						:     Union[str, Any]							     =  self.get_rust_tokenizer()
       lowercase_						:     Tuple							     =  tokenizer.encode(a      )
       lowercase_						:     Optional[int]							     =  rust_tokenizer.encode(a      )
       self.assertListEqual(a	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      )	,  ["ah", "\u535A", "\u63A8", "zz"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["h\u00E9llo"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      )	,  ["hello"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       lowercase_						:     str							     =  BasicTokenizer(do_lower_case=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     int							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HäLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Union[str, Any]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  BasicTokenizer(do_lower_case=a	,  strip_accents=a      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      )	,  ["HaLLo", "!", "how", "Are", "yoU", "?"]      )




    def 					lowerCAmelCase__					(      self	:							Optional[Any]      ):






       '''simple docstring'''






       lowercase_						:     Any							     =  BasicTokenizer(do_lower_case=a	,  never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      )	,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )




    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Union[str, Any]							     =  BasicTokenizer()
       lowercase_						:     Optional[int]							     =  "a\n'll !!to?'d of, can't."
       lowercase_						:     List[Any]							     =  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(a      )	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase_						:     int							     =  {}
       for i, token in enumerate(a      ):
          lowercase_						:     Any							     =  i
       lowercase_						:     Union[str, Any]							     =  WordpieceTokenizer(vocab=a	,  unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      )	,  []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      )	,  ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      )	,  ["[UNK]", "runn", "##ing"]      )




    def 					lowerCAmelCase__					(      self	:							List[str]      ):






       '''simple docstring'''






       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]      ):






       '''simple docstring'''






       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )




    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  self.get_tokenizer()
       lowercase_						:     Optional[Any]							     =  self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )

       self.assertListEqual(
           [rust_tokenizer.tokenize(a      ) for t in ["Test", "\xad", "test"]]	,  [["[UNK]"], [], ["[UNK]"]]      )




    @slow
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     str							     =  self.tokenizer_class.from_pretrained("bert-base-uncased"      )

       lowercase_						:     Tuple							     =  tokenizer.encode("sequence builders"	,  add_special_tokens=a      )
       lowercase_						:     Union[str, Any]							     =  tokenizer.encode("multi-sequence build"	,  add_special_tokens=a      )

       lowercase_						:     Dict							     =  tokenizer.build_inputs_with_special_tokens(a      )
       lowercase_						:     Optional[Any]							     =  tokenizer.build_inputs_with_special_tokens(a	,  a      )

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Dict							     =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             lowercase_						:     int							     =  tokenizer_r.encode_plus(
                 a	,  return_attention_mask=a	,  return_token_type_ids=a	,  return_offsets_mapping=a	,  add_special_tokens=a	,  )

             lowercase_						:     Tuple							     =  tokenizer_r.do_lower_case if hasattr(a	,  "do_lower_case"      ) else False
             lowercase_						:     List[Any]							     =  (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]	,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
             self.assertEqual([e[0] for e in expected_results]	,  tokens["offset_mapping"]      )





    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       lowercase_						:     Tuple							     =  ["的", "人", "有"]
       lowercase_						:     int							     =  "".join(a      )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
             lowercase_						:     str							     =  True
             lowercase_						:     Dict							     =  self.tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )

             lowercase_						:     str							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )

             lowercase_						:     Tuple							     =  False
             lowercase_						:     Union[str, Any]							     =  self.rust_tokenizer_class.from_pretrained(a	,  **a      )
             lowercase_						:     List[str]							     =  self.tokenizer_class.from_pretrained(a	,  **a      )

             lowercase_						:     Union[str, Any]							     =  tokenizer_r.encode(a	,  add_special_tokens=a      )
             lowercase_						:     Optional[Any]							     =  tokenizer_p.encode(a	,  add_special_tokens=a      )

             lowercase_						:     int							     =  tokenizer_r.convert_ids_to_tokens(a      )
             lowercase_						:     List[Any]							     =  tokenizer_p.convert_ids_to_tokens(a      )

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase_						:     int							     =  [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(a      )
             ]
             self.assertListEqual(a	,  a      )
             self.assertListEqual(a	,  a      )




