





'''simple docstring'''

from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a   	=    logging.get_logger(__name__)

# TODO Update this
a   	=    {
    "facebook/esm-1b": "https://huggingface.co/facebook/esm-1b/resolve/main/config.json",
    # See all ESM models at https://huggingface.co/models?filter=esm
}







class __a			( _snake_case      ):
   __UpperCamelCase :			str						   =						'esm'





   def __init__(       self					:      Tuple					,lowerCamelCase					:      List[Any]=None					,lowerCamelCase					:      str=None					,lowerCamelCase					:      Any=None					,lowerCamelCase					:      Union[str, Any]=768					,lowerCamelCase					:      Tuple=12					,lowerCamelCase					:      int=12					,lowerCamelCase					:      Optional[int]=3072					,lowerCamelCase					:      List[Any]=0.1					,lowerCamelCase					:      Optional[int]=0.1					,lowerCamelCase					:      Any=1026					,lowerCamelCase					:      str=0.02					,lowerCamelCase					:      int=1E-1_2					,lowerCamelCase					:      Union[str, Any]="absolute"					,lowerCamelCase					:      Optional[Any]=True					,lowerCamelCase					:      str=None					,lowerCamelCase					:      Optional[int]=False					,lowerCamelCase					:      int=False					,lowerCamelCase					:      Union[str, Any]=None					,lowerCamelCase					:      Any=None					,**lowerCamelCase					:      Any					,):

      '''simple docstring'''

      super().__init__(pad_token_id=lowerCamelCase					,mask_token_id=lowerCamelCase					,**lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							vocab_size
      __SCREAMING_SNAKE_CASE   				=							hidden_size
      __SCREAMING_SNAKE_CASE   				=							num_hidden_layers
      __SCREAMING_SNAKE_CASE   				=							num_attention_heads
      __SCREAMING_SNAKE_CASE   				=							intermediate_size
      __SCREAMING_SNAKE_CASE   				=							hidden_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							attention_probs_dropout_prob
      __SCREAMING_SNAKE_CASE   				=							max_position_embeddings
      __SCREAMING_SNAKE_CASE   				=							initializer_range
      __SCREAMING_SNAKE_CASE   				=							layer_norm_eps
      __SCREAMING_SNAKE_CASE   				=							position_embedding_type
      __SCREAMING_SNAKE_CASE   				=							use_cache
      __SCREAMING_SNAKE_CASE   				=							emb_layer_norm_before
      __SCREAMING_SNAKE_CASE   				=							token_dropout
      __SCREAMING_SNAKE_CASE   				=							is_folding_model
      if is_folding_model:
         if esmfold_config is None:
            logger.info("""No esmfold_config supplied for folding model, using default values."""     )
            __SCREAMING_SNAKE_CASE   				=							EsmFoldConfig()
         elif isinstance(lowerCamelCase					,lowerCamelCase     ):
            __SCREAMING_SNAKE_CASE   				=							EsmFoldConfig(**lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							esmfold_config
         if vocab_list is None:
            logger.warning("""No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!"""     )
            __SCREAMING_SNAKE_CASE   				=							get_default_vocab_list()
         else:
            __SCREAMING_SNAKE_CASE   				=							vocab_list
      else:
         __SCREAMING_SNAKE_CASE   				=							None
         __SCREAMING_SNAKE_CASE   				=							None
      if self.esmfold_config is not None and getattr(self.esmfold_config					,"""use_esm_attn_map"""					,lowerCamelCase     ):
         raise ValueError("""The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!"""     )





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							super().to_dict()
      if isinstance(self.esmfold_config					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							self.esmfold_config.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			str						   =						None
   __UpperCamelCase :			bool						   =						True
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			float						   =						0

   __UpperCamelCase :			bool						   =						True
   __UpperCamelCase :			bool						   =						False

   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			"TrunkConfig"						   =						None





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      if self.trunk is None:
         __SCREAMING_SNAKE_CASE   				=							TrunkConfig()
      elif isinstance(self.trunk					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							TrunkConfig(**self.trunk     )





   def   UpperCAmelCase__  (       self					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							asdict(self     )
      __SCREAMING_SNAKE_CASE   				=							self.trunk.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			int						   =						48
   __UpperCamelCase :			int						   =						1024
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			int						   =						32
   __UpperCamelCase :			float						   =						0
   __UpperCamelCase :			float						   =						0
   __UpperCamelCase :			bool						   =						False
   __UpperCamelCase :			int						   =						4
   __UpperCamelCase :			Optional[int]						   =						128
   __UpperCamelCase :			"StructureModuleConfig"						   =						None





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      if self.structure_module is None:
         __SCREAMING_SNAKE_CASE   				=							StructureModuleConfig()
      elif isinstance(self.structure_module					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							StructureModuleConfig(**self.structure_module     )

      if self.max_recycles <= 0:
         raise ValueError(f"""`max_recycles` should be positive, got {self.max_recycles}."""     )
      if self.sequence_state_dim % self.sequence_state_dim != 0:
         raise ValueError(
             """`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"""
             f""" {self.sequence_state_dim} and {self.sequence_state_dim}."""     )
      if self.pairwise_state_dim % self.pairwise_state_dim != 0:
         raise ValueError(
             """`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"""
             f""" {self.pairwise_state_dim} and {self.pairwise_state_dim}."""     )

      __SCREAMING_SNAKE_CASE   				=							self.sequence_state_dim // self.sequence_head_width
      __SCREAMING_SNAKE_CASE   				=							self.pairwise_state_dim // self.pairwise_head_width

      if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
         raise ValueError(
             """`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"""
             f""" {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}."""     )
      if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
         raise ValueError(
             """`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"""
             f""" {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}."""     )
      if self.pairwise_state_dim % 2 != 0:
         raise ValueError(f"""`pairwise_state_dim` should be even, got {self.pairwise_state_dim}."""     )

      if self.dropout >= 0.4:
         raise ValueError(f"""`dropout` should not be greater than 0.4, got {self.dropout}."""     )





   def   UpperCAmelCase__  (       self					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							asdict(self     )
      __SCREAMING_SNAKE_CASE   				=							self.structure_module.to_dict()
      return output







@dataclass
class __a			:
   __UpperCamelCase :			int						   =						384
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						16
   __UpperCamelCase :			int						   =						128
   __UpperCamelCase :			int						   =						12
   __UpperCamelCase :			int						   =						4
   __UpperCamelCase :			int						   =						8
   __UpperCamelCase :			float						   =						0.1
   __UpperCamelCase :			int						   =						8
   __UpperCamelCase :			int						   =						1
   __UpperCamelCase :			int						   =						2
   __UpperCamelCase :			int						   =						7
   __UpperCamelCase :			int						   =						10
   __UpperCamelCase :			float						   =						1E-8
   __UpperCamelCase :			float						   =						1E5





   def   UpperCAmelCase__  (       self					:      List[Any]     ):

      '''simple docstring'''

      return asdict(self     )





def     __magic_name__							(				)    ->       Dict:

   '''simple docstring'''







   return (
       "<cls>",
       "<pad>",
       "<eos>",
       "<unk>",
       "L",
       "A",
       "G",
       "V",
       "S",
       "E",
       "R",
       "T",
       "I",
       "D",
       "P",
       "K",
       "Q",
       "N",
       "F",
       "Y",
       "M",
       "H",
       "W",
       "C",
       "X",
       "B",
       "U",
       "Z",
       "O",
       ".",
       "-",
       "<null_1>",
       "<mask>",
   )











"""simple docstring"""






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class lowerCAmelCase__      (	lowercase,      unittest.TestCase		):






			'''simple docstring'''



			lowerCamelCase__ =     RobertaTokenizer
			lowerCamelCase__ =     RobertaTokenizerFast
			lowerCamelCase__ =     True
			lowerCamelCase__ =     {"""cls_token""": """<s>"""}
			def    A_       (    self    ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							_lowerCamelCase	:     Optional[Any]       =       [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							]
							_lowerCamelCase	:     Optional[int]       =       dict(zip(lowercase    , range(len(lowercase    )    )    )    )
							_lowerCamelCase	:     Optional[int]       =       ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							_lowerCamelCase	:     Optional[Any]       =       {'unk_token': '<unk>'}

							_lowerCamelCase	:     Optional[int]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['vocab_file']    )
							_lowerCamelCase	:     Optional[Any]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['merges_file']    )
							with open(self.vocab_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write(json.dumps(lowercase    ) + '\n'    )
							with open(self.merges_file    , 'w'    , encoding='utf-8'    ) as fp:
											fp.write('\n'.join(lowercase    )    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return self.tokenizer_class.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , **lowercase    ):
							kwargs.update(self.special_tokens_map    )
							return RobertaTokenizerFast.from_pretrained(self.tmpdirname    , **lowercase    )
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Any       =       'lower newer'
							_lowerCamelCase	:     Union[str, Any]       =       'lower newer'
							return input_text, output_text
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class(self.vocab_file    , self.merges_file    , **self.special_tokens_map    )
							_lowerCamelCase	:     List[str]       =       'lower newer'
							_lowerCamelCase	:     Tuple       =       ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
							_lowerCamelCase	:     List[str]       =       tokenizer.tokenize(lowercase    )  # , add_prefix_space=True)
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokens + [tokenizer.unk_token]
							_lowerCamelCase	:     str       =       [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )    , lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('Hello world!'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 2]    )
							self.assertListEqual(
							    tokenizer.encode('Hello world! cécé herlolip 418'    , add_special_tokens=lowercase    )    , [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]    , )
			@slow
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained('roberta-base'    )

							_lowerCamelCase	:     Tuple       =       tokenizer.encode('sequence builders'    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.encode('multi-sequence build'    , add_special_tokens=lowercase    )

							_lowerCamelCase	:     List[Any]       =       tokenizer.encode(
							    'sequence builders'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[str]       =       tokenizer.encode(
							    'sequence builders'    , 'multi-sequence build'    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )

							_lowerCamelCase	:     Dict       =       tokenizer.build_inputs_with_special_tokens(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       tokenizer.build_inputs_with_special_tokens(lowercase    , lowercase    )

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode
			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       self.get_tokenizer()

							_lowerCamelCase	:     str       =       'Encode this sequence.'
							_lowerCamelCase	:     Optional[int]       =       tokenizer.byte_encoder[' '.encode('utf-8'    )[0]]

							# Testing encoder arguments
							_lowerCamelCase	:     Optional[int]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     List[str]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    , add_prefix_space=lowercase    )
							_lowerCamelCase	:     List[Any]       =       tokenizer.convert_ids_to_tokens(encoded[0]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							tokenizer.add_special_tokens({'bos_token': '<s>'}    )
							_lowerCamelCase	:     Optional[Any]       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.convert_ids_to_tokens(encoded[1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )

							# Testing spaces after special tokens
							_lowerCamelCase	:     List[Any]       =       '<mask>'
							tokenizer.add_special_tokens(
							    {'mask_token': AddedToken(lowercase    , lstrip=lowercase    , rstrip=lowercase    )}    )  # mask token has a left space
							_lowerCamelCase	:     List[str]       =       tokenizer.convert_tokens_to_ids(lowercase    )

							_lowerCamelCase	:     Optional[int]       =       'Encode <mask> sequence'
							_lowerCamelCase	:     int       =       'Encode <mask>sequence'

							_lowerCamelCase	:     str       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Tuple       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Any       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Union[str, Any]       =       encoded.index(lowercase    )
							_lowerCamelCase	:     int       =       tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]    )[0]
							self.assertNotEqual(lowercase    , lowercase    )
			def    A_       (    self    ):
							pass
			def    A_       (    self    ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     Dict       =       self.tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     List[str]       =       'A, <mask> AllenNLP sentence.'
															_lowerCamelCase	:     List[Any]       =       tokenizer_r.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_p.encode_plus(lowercase    , add_special_tokens=lowercase    , return_token_type_ids=lowercase    )

															# token_type_ids should put 0 everywhere
															self.assertEqual(sum(tokens_r['token_type_ids']    )    , sum(tokens_p['token_type_ids']    )    )

															# attention_mask should put 1 everywhere, so sum over length should be 1
															self.assertEqual(
															    sum(tokens_r['attention_mask']    ) / len(tokens_r['attention_mask']    )    , sum(tokens_p['attention_mask']    ) / len(tokens_p['attention_mask']    )    , )

															_lowerCamelCase	:     List[Any]       =       tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']    )
															_lowerCamelCase	:     Tuple       =       tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']    )

															# Rust correctly handles the space before the mask while python doesnt
															self.assertSequenceEqual(tokens_p['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )
															self.assertSequenceEqual(tokens_r['input_ids']    , [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]    )

															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
															self.assertSequenceEqual(
															    lowercase    , ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']    )
			def    A_       (    self    ):
							for trim_offsets, add_prefix_space in itertools.product([True, False]    , repeat=2    ):
											_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )

											_lowerCamelCase	:     Optional[int]       =       json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()    )
											_lowerCamelCase	:     Dict       =       json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()    )

											self.assertEqual(pre_tokenizer_state['add_prefix_space']    , lowercase    )

											self.assertEqual(post_processor_state['add_prefix_space']    , lowercase    )
											self.assertEqual(post_processor_state['trim_offsets']    , lowercase    )






			def    A_       (    self    ):
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     Dict       =       'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
															_lowerCamelCase	:     Tuple       =       F'''{text_of_1_token} {text_of_1_token}'''

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ) + 1, len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     List[str]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     int       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (len(lowercase    ), len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     int       =       F''' {text}'''

															# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
															#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
															# )
															# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
															# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
															# self.assertEqual(
															#     encoding.offset_mapping[1],
															#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
															# )

															_lowerCamelCase	:     Optional[Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[int]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (1, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ) + 1, 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Optional[int]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Union[str, Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )

															_lowerCamelCase	:     Union[str, Any]       =       self.rust_tokenizer_class.from_pretrained(
															    lowercase    , use_fast=lowercase    , add_prefix_space=lowercase    , trim_offsets=lowercase    )
															_lowerCamelCase	:     Optional[Any]       =       tokenizer_r(lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    )
															self.assertEqual(encoding.offset_mapping[0]    , (0, 1 + len(lowercase    ))    )
															self.assertEqual(
															    encoding.offset_mapping[1]    , (1 + len(lowercase    ), 1 + len(lowercase    ) + 1 + len(lowercase    ))    , )