
'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__						       =      get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
@require_tokenizers
class 	snake_case__       (      __SCREAMING_SNAKE_CASE    ,					unittest.TestCase					):







		"""simple docstring"""

		lowerCamelCase     							=						ReformerTokenizer
		lowerCamelCase     							=						ReformerTokenizerFast
		lowerCamelCase     							=						True
		lowerCamelCase     							=						False
		lowerCamelCase     							=						True



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Dict:






				"""simple docstring"""







				super().setUp()

				snake_case  :       int     			=						ReformerTokenizer(UpperCamelCase__  ,	keep_accents=UpperCamelCase__      )
				tokenizer.save_pretrained(self.tmpdirname      )



		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			Tuple:






				"""simple docstring"""







				snake_case  :       int     			=						'''<s>'''
				snake_case  :       str     			=						1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__      )  ,	UpperCamelCase__      )
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__      )  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			Tuple      )						->			str:






				"""simple docstring"""







				snake_case  :       Tuple     			=						list(self.get_tokenizer().get_vocab().keys()      )

				self.assertEqual(vocab_keys[0]  ,	'''<unk>'''      )
				self.assertEqual(vocab_keys[1]  ,	'''<s>'''      )
				self.assertEqual(vocab_keys[-1]  ,	'''j'''      )
				self.assertEqual(len(UpperCamelCase__      )  ,	1000      )



		def       lowerCAmelCase				(      self				:			Tuple      )						->			Any:






				"""simple docstring"""







				self.assertEqual(self.get_tokenizer().vocab_size  ,	1000      )



		def       lowerCAmelCase				(      self				:			Any      )						->			List[Any]:






				"""simple docstring"""







				if not self.test_rust_tokenizer:
						return

				snake_case  :       Tuple     			=						self.get_tokenizer()
				snake_case  :       int     			=						self.get_rust_tokenizer()

				snake_case  :       str     			=						'''I was born in 92000, and this is falsé.'''

				snake_case  :       int     			=						tokenizer.tokenize(UpperCamelCase__      )
				snake_case  :       str     			=						rust_tokenizer.tokenize(UpperCamelCase__      )
				self.assertListEqual(UpperCamelCase__  ,	UpperCamelCase__      )

				snake_case  :       Optional[Any]     			=						tokenizer.encode(UpperCamelCase__  ,	add_special_tokens=UpperCamelCase__      )
				snake_case  :       str     			=						rust_tokenizer.encode(UpperCamelCase__  ,	add_special_tokens=UpperCamelCase__      )
				self.assertListEqual(UpperCamelCase__  ,	UpperCamelCase__      )

				snake_case  :       str     			=						self.get_rust_tokenizer()
				snake_case  :       int     			=						tokenizer.encode(UpperCamelCase__      )
				snake_case  :       Any     			=						rust_tokenizer.encode(UpperCamelCase__      )
				self.assertListEqual(UpperCamelCase__  ,	UpperCamelCase__      )



		def       lowerCAmelCase				(      self				:			List[str]  ,	UpperCamelCase__				:			List[str]=15      )						->			List[Any]:






				"""simple docstring"""







				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
								snake_case  :       Optional[Any]     			=						self.rust_tokenizer_class.from_pretrained(UpperCamelCase__  ,	**UpperCamelCase__      )

								# Simple input
								snake_case  :       Union[str, Any]     			=						'''This is a simple input'''
								snake_case  :       str     			=						['''This is a simple input 1''', '''This is a simple input 2''']
								snake_case  :       List[Any]     			=						('''This is a simple input''', '''This is a pair''')
								snake_case  :       List[str]     			=						[
								    ('''This is a simple input 1''', '''This is a simple input 2'''),
								    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
								]

								# Simple input tests
								self.assertRaises(UpperCamelCase__  ,	tokenizer_r.encode  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''      )

								# Simple input
								self.assertRaises(UpperCamelCase__  ,	tokenizer_r.encode_plus  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''      )

								# Simple input
								self.assertRaises(
								    UpperCamelCase__  ,	tokenizer_r.batch_encode_plus  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''  ,	)

								# Pair input
								self.assertRaises(UpperCamelCase__  ,	tokenizer_r.encode  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''      )

								# Pair input
								self.assertRaises(UpperCamelCase__  ,	tokenizer_r.encode_plus  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''      )

								# Pair input
								self.assertRaises(
								    UpperCamelCase__  ,	tokenizer_r.batch_encode_plus  ,	UpperCamelCase__  ,	max_length=UpperCamelCase__  ,	padding='''max_length'''  ,	)



		def       lowerCAmelCase				(      self				:			List[Any]      )						->			Optional[Any]:






				"""simple docstring"""







				pass



		def       lowerCAmelCase				(      self				:			str      )						->			Dict:






				"""simple docstring"""







				snake_case  :       List[Any]     			=						ReformerTokenizer(UpperCamelCase__  ,	keep_accents=UpperCamelCase__      )

				snake_case  :       Tuple     			=						tokenizer.tokenize('''This is a test'''      )
				self.assertListEqual(UpperCamelCase__  ,	['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(UpperCamelCase__      )  ,	[285, 46, 10, 170, 382]  ,	)

				snake_case  :       List[Any]     			=						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
				self.assertListEqual(
				    UpperCamelCase__  ,	[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''9''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''é''',
				        '''.''',
				    ]  ,	)
				snake_case  :       Tuple     			=						tokenizer.convert_tokens_to_ids(UpperCamelCase__      )
				self.assertListEqual(
				    UpperCamelCase__  ,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]  ,	)

				snake_case  :       Union[str, Any]     			=						tokenizer.convert_ids_to_tokens(UpperCamelCase__      )
				self.assertListEqual(
				    UpperCamelCase__  ,	[
				        SPIECE_UNDERLINE + '''I''',
				        SPIECE_UNDERLINE + '''was''',
				        SPIECE_UNDERLINE + '''b''',
				        '''or''',
				        '''n''',
				        SPIECE_UNDERLINE + '''in''',
				        SPIECE_UNDERLINE + '''''',
				        '''<unk>''',
				        '''2''',
				        '''0''',
				        '''0''',
				        '''0''',
				        ''',''',
				        SPIECE_UNDERLINE + '''and''',
				        SPIECE_UNDERLINE + '''this''',
				        SPIECE_UNDERLINE + '''is''',
				        SPIECE_UNDERLINE + '''f''',
				        '''al''',
				        '''s''',
				        '''<unk>''',
				        '''.''',
				    ]  ,	)



		@cached_property
		def       lowerCAmelCase				(      self				:			List[Any]      )						->			Dict:






				"""simple docstring"""







				return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''      )



		@slow
		def       lowerCAmelCase				(      self				:			Any      )						->			Optional[int]:






				"""simple docstring"""







				snake_case  :       Tuple     			=						'''Hello World!'''
				snake_case  :       List[str]     			=						[126, 32, 262, 152, 38, 72, 287]

				self.assertListEqual(UpperCamelCase__  ,	self.big_tokenizer.encode(UpperCamelCase__      )      )



		@slow
		def       lowerCAmelCase				(      self				:			Tuple      )						->			Optional[int]:






				"""simple docstring"""







				snake_case  :       Any     			=						(
				    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
				    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
				)
				snake_case  :       List[Any]     			=						[
				    108,
				    265,
				    24,
				    111,
				    4,
				    258,
				    156,
				    35,
				    28,
				    275,
				    3,
				    259,
				    297,
				    260,
				    84,
				    4,
				    35,
				    110,
				    44,
				    8,
				    259,
				    91,
				    268,
				    21,
				    11,
				    209,
				    274,
				    109,
				    266,
				    277,
				    117,
				    86,
				    93,
				    315,
				    258,
				    278,
				    258,
				    277,
				    258,
				    0,
				    258,
				    288,
				    258,
				    319,
				    258,
				    0,
				    258,
				    0,
				    258,
				    0,
				    258,
				    0,
				    258,
				    287,
				    258,
				    315,
				    258,
				    289,
				    258,
				    278,
				    99,
				    269,
				    266,
				    262,
				    8,
				    259,
				    241,
				    4,
				    217,
				    230,
				    268,
				    266,
				    55,
				    168,
				    106,
				    75,
				    193,
				    266,
				    223,
				    27,
				    49,
				    26,
				    282,
				    25,
				    264,
				    299,
				    19,
				    26,
				    0,
				    258,
				    277,
				    117,
				    86,
				    93,
				    176,
				    183,
				    270,
				    11,
				    262,
				    42,
				    61,
				    265,
				]

				self.assertListEqual(UpperCamelCase__  ,	self.big_tokenizer.encode(UpperCamelCase__      )      )



		@require_torch
		@slow
		def       lowerCAmelCase				(      self				:			Optional[Any]      )						->			int:






				"""simple docstring"""







				import torch

				from transformers import ReformerConfig, ReformerModel

				# Build sequence
				snake_case  :       Tuple     			=						list(self.big_tokenizer.get_vocab().keys()      )[:10]
				snake_case  :       Optional[Any]     			=						''' '''.join(UpperCamelCase__      )
				snake_case  :       Dict     			=						self.big_tokenizer.encode_plus(UpperCamelCase__  ,	return_tensors='''pt'''      )
				snake_case  :       Union[str, Any]     			=						self.big_tokenizer.batch_encode_plus([sequence, sequence]  ,	return_tensors='''pt'''      )

				snake_case  :       Union[str, Any]     			=						ReformerConfig()
				# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
				snake_case  :       Optional[Any]     			=						encoded_sequence['''input_ids'''].shape
				snake_case  :       Any     			=						ReformerModel(UpperCamelCase__      )

				# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
				assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

				with torch.no_grad():
						model(**UpperCamelCase__      )
						model(**UpperCamelCase__      )



		@slow
		def       lowerCAmelCase				(      self				:			Dict      )						->			Any:






				"""simple docstring"""







				snake_case  :       Any     			=						{'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
				# fmt: on

				# This tokenizer does not know some characters like ")".
				# That is the reason why we use very simple texts here.
				# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
				snake_case  :       Tuple     			=						[
				    '''This is a very simple sentence.''',
				    '''The quick brown fox jumps over the lazy dog.''',
				]

				self.tokenizer_integration_test_util(
				    expected_encoding=UpperCamelCase__  ,	model_name='''google/reformer-crime-and-punishment'''  ,	revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''  ,	padding=UpperCamelCase__  ,	sequences=UpperCamelCase__  ,	)




'''simple docstring'''
import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowercase__						       =      get_logger(__name__)






class 	snake_case__       (      enum.Enum					):







		"""simple docstring"""

		lowerCamelCase     							=						"""all_checks"""
		lowerCamelCase     							=						"""basic_checks"""
		lowerCamelCase     							=						"""no_checks"""







class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__=None			)  ->							str:


		'''simple docstring'''


		if expected_checksums is None:
				logger.info('''Unable to verify checksums.'''			)
				return
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise ExpectedMoreDownloadedFiles(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise UnexpectedDownloadedFile(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		snake_case  :       Any     			=						[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
		snake_case  :       Union[str, Any]     			=						''' for ''' + verification_name if verification_name is not None else ''''''
		if len(SCREAMING_SNAKE_CASE__			) > 0:
				raise NonMatchingChecksumError(
				    F'Checksums didn\'t match{for_verification_name}:\n'
				    F'{bad_urls}\n'
				    '''Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'''			)
		logger.info('''All the checksums matched successfully''' + for_verification_name			)






class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)  ->							List[str]:


		'''simple docstring'''


		if expected_splits is None:
				logger.info('''Unable to verify splits sizes.'''			)
				return
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise ExpectedMoreSplits(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise UnexpectedSplits(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		snake_case  :       Dict     			=						[
		    {'''expected''': expected_splits[name], '''recorded''': recorded_splits[name]}
		    for name in expected_splits
		    if expected_splits[name].num_examples != recorded_splits[name].num_examples
		]
		if len(SCREAMING_SNAKE_CASE__			) > 0:
				raise NonMatchingSplitsSizesError(str(SCREAMING_SNAKE_CASE__			)			)
		logger.info('''All the splits matched successfully.'''			)




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ = True			)  ->							dict:


		'''simple docstring'''


		if record_checksum:
				snake_case  :       Any     			=						shaaaa()
				with open(SCREAMING_SNAKE_CASE__ ,       '''rb'''			) as f:
						for chunk in iter(lambda: f.read(1 << 20			) ,       B''''''			):
								m.update(SCREAMING_SNAKE_CASE__			)
						snake_case  :       Any     			=						m.hexdigest()
		else:
				snake_case  :       Tuple     			=						None
		return {"num_bytes": os.path.getsize(SCREAMING_SNAKE_CASE__			), "checksum": checksum}




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__			)  ->							Dict:


		'''simple docstring'''


		if dataset_size and config.IN_MEMORY_MAX_SIZE:
				return dataset_size < config.IN_MEMORY_MAX_SIZE
		else:
				return False


