


'''simple docstring'''

import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase     :							Dict    =				get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class 		snake_case				(    _a				,    unittest.TestCase						):

				"""simple docstring"""




				_lowerCAmelCase           =    ReformerTokenizer
				_lowerCAmelCase           =    ReformerTokenizerFast
				_lowerCAmelCase           =    True
				_lowerCAmelCase           =    False
				_lowerCAmelCase           =    True





				def    lowercase__							(	self )    ->      Dict:







					"""simple docstring"""






					super().setUp()

					snake_case__			:			Any             =		ReformerTokenizer(A_       ,    keep_accents=A_ )
					tokenizer.save_pretrained(self.tmpdirname )





				def    lowercase__							(	self )    ->      str:







					"""simple docstring"""






					snake_case__			:			str             =		'''<s>'''
					snake_case__			:			int             =		1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_ )       ,    A_ )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_ )       ,    A_ )





				def    lowercase__							(	self )    ->      List[str]:







					"""simple docstring"""






					snake_case__			:			Tuple             =		list(self.get_tokenizer().get_vocab().keys() )

					self.assertEqual(vocab_keys[0]       ,    '''<unk>''' )
					self.assertEqual(vocab_keys[1]       ,    '''<s>''' )
					self.assertEqual(vocab_keys[-1]       ,    '''j''' )
					self.assertEqual(len(A_ )       ,    1000 )





				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					self.assertEqual(self.get_tokenizer().vocab_size       ,    1000 )





				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					if not self.test_rust_tokenizer:
						return

					snake_case__			:			str             =		self.get_tokenizer()
					snake_case__			:			List[str]             =		self.get_rust_tokenizer()

					snake_case__			:			Union[str, Any]             =		'''I was born in 92000, and this is falsé.'''

					snake_case__			:			List[Any]             =		tokenizer.tokenize(A_ )
					snake_case__			:			Any             =		rust_tokenizer.tokenize(A_ )
					self.assertListEqual(A_       ,    A_ )

					snake_case__			:			Optional[int]             =		tokenizer.encode(A_       ,    add_special_tokens=A_ )
					snake_case__			:			Tuple             =		rust_tokenizer.encode(A_       ,    add_special_tokens=A_ )
					self.assertListEqual(A_       ,    A_ )

					snake_case__			:			Dict             =		self.get_rust_tokenizer()
					snake_case__			:			Dict             =		tokenizer.encode(A_ )
					snake_case__			:			Tuple             =		rust_tokenizer.encode(A_ )
					self.assertListEqual(A_       ,    A_ )





				def    lowercase__							(	self       ,    lowerCamelCase=15 )    ->      Union[str, Any]:







					"""simple docstring"""






					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
							snake_case__			:			Union[str, Any]             =		self.rust_tokenizer_class.from_pretrained(A_       ,    **A_ )

							# Simple input
							snake_case__			:			Tuple             =		'''This is a simple input'''
							snake_case__			:			Tuple             =		['''This is a simple input 1''', '''This is a simple input 2''']
							snake_case__			:			List[Any]             =		('''This is a simple input''', '''This is a pair''')
							snake_case__			:			Any             =		[
							    ('''This is a simple input 1''', '''This is a simple input 2'''),
							    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
							]

							# Simple input tests
							self.assertRaises(A_       ,    tokenizer_r.encode       ,    A_       ,    max_length=A_       ,    padding='''max_length''' )

							# Simple input
							self.assertRaises(A_       ,    tokenizer_r.encode_plus       ,    A_       ,    max_length=A_       ,    padding='''max_length''' )

							# Simple input
							self.assertRaises(
							    A_       ,    tokenizer_r.batch_encode_plus       ,    A_       ,    max_length=A_       ,    padding='''max_length'''       ,    )

							# Pair input
							self.assertRaises(A_       ,    tokenizer_r.encode       ,    A_       ,    max_length=A_       ,    padding='''max_length''' )

							# Pair input
							self.assertRaises(A_       ,    tokenizer_r.encode_plus       ,    A_       ,    max_length=A_       ,    padding='''max_length''' )

							# Pair input
							self.assertRaises(
							    A_       ,    tokenizer_r.batch_encode_plus       ,    A_       ,    max_length=A_       ,    padding='''max_length'''       ,    )





				def    lowercase__							(	self )    ->      Dict:







					"""simple docstring"""






					pass





				def    lowercase__							(	self )    ->      Union[str, Any]:







					"""simple docstring"""






					snake_case__			:			List[str]             =		ReformerTokenizer(A_       ,    keep_accents=A_ )

					snake_case__			:			Optional[int]             =		tokenizer.tokenize('''This is a test''' )
					self.assertListEqual(A_       ,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(A_ )       ,    [285, 46, 10, 170, 382]       ,    )

					snake_case__			:			Tuple             =		tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
					self.assertListEqual(
					    A_       ,    [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ]       ,    )
					snake_case__			:			List[Any]             =		tokenizer.convert_tokens_to_ids(A_ )
					self.assertListEqual(
					    A_       ,    [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,    )

					snake_case__			:			int             =		tokenizer.convert_ids_to_tokens(A_ )
					self.assertListEqual(
					    A_       ,    [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ]       ,    )





				@cached_property
				def    lowercase__							(	self )    ->      int:







					"""simple docstring"""






					return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment''' )





				@slow
				def    lowercase__							(	self )    ->      Any:







					"""simple docstring"""






					snake_case__			:			Tuple             =		'''Hello World!'''
					snake_case__			:			Dict             =		[126, 32, 262, 152, 38, 72, 287]

					self.assertListEqual(A_       ,    self.big_tokenizer.encode(A_ ) )





				@slow
				def    lowercase__							(	self )    ->      Union[str, Any]:







					"""simple docstring"""






					snake_case__			:			List[str]             =		(
					    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will'''
					    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
					)
					snake_case__			:			int             =		[
					    108,
					    265,
					    24,
					    111,
					    4,
					    258,
					    156,
					    35,
					    28,
					    275,
					    3,
					    259,
					    297,
					    260,
					    84,
					    4,
					    35,
					    110,
					    44,
					    8,
					    259,
					    91,
					    268,
					    21,
					    11,
					    209,
					    274,
					    109,
					    266,
					    277,
					    117,
					    86,
					    93,
					    315,
					    258,
					    278,
					    258,
					    277,
					    258,
					    0,
					    258,
					    288,
					    258,
					    319,
					    258,
					    0,
					    258,
					    0,
					    258,
					    0,
					    258,
					    0,
					    258,
					    287,
					    258,
					    315,
					    258,
					    289,
					    258,
					    278,
					    99,
					    269,
					    266,
					    262,
					    8,
					    259,
					    241,
					    4,
					    217,
					    230,
					    268,
					    266,
					    55,
					    168,
					    106,
					    75,
					    193,
					    266,
					    223,
					    27,
					    49,
					    26,
					    282,
					    25,
					    264,
					    299,
					    19,
					    26,
					    0,
					    258,
					    277,
					    117,
					    86,
					    93,
					    176,
					    183,
					    270,
					    11,
					    262,
					    42,
					    61,
					    265,
					]

					self.assertListEqual(A_       ,    self.big_tokenizer.encode(A_ ) )





				@require_torch
				@slow
				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					import torch

					from transformers import ReformerConfig, ReformerModel

					# Build sequence
					snake_case__			:			int             =		list(self.big_tokenizer.get_vocab().keys() )[:10]
					snake_case__			:			str             =		''' '''.join(A_ )
					snake_case__			:			Dict             =		self.big_tokenizer.encode_plus(A_       ,    return_tensors='''pt''' )
					snake_case__			:			Union[str, Any]             =		self.big_tokenizer.batch_encode_plus([sequence, sequence]       ,    return_tensors='''pt''' )

					snake_case__			:			List[str]             =		ReformerConfig()
					# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
					snake_case__			:			Tuple             =		encoded_sequence['''input_ids'''].shape
					snake_case__			:			Tuple             =		ReformerModel(A_ )

					# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
					assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

					with torch.no_grad():
						model(**A_ )
						model(**A_ )





				@slow
				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					snake_case__			:			Optional[int]             =		{'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					# This tokenizer does not know some characters like ")".
					# That is the reason why we use very simple texts here.
					# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
					snake_case__			:			str             =		[
					    '''This is a very simple sentence.''',
					    '''The quick brown fox jumps over the lazy dog.''',
					]

					self.tokenizer_integration_test_util(
					    expected_encoding=A_       ,    model_name='''google/reformer-crime-and-punishment'''       ,    revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''       ,    padding=A_       ,    sequences=A_       ,    )









"""simple docstring"""

import pytest
import requests

from datasets.utils.file_utils import http_head

from .utils import OfflineSimulationMode, RequestWouldHangIndefinitelyError, offline


@pytest.mark.integration
def lowerCAmelCase_   (					):






 """simple docstring"""







 with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT			):
  with pytest.raises(UpperCamelCase__			):
   requests.request("""GET"""					, """https://huggingface.co"""			)
  with pytest.raises(requests.exceptions.ConnectTimeout			):
   requests.request("""GET"""					, """https://huggingface.co"""					, timeout=1.0			)


@pytest.mark.integration
def lowerCAmelCase_   (					):






 """simple docstring"""







 with offline(OfflineSimulationMode.CONNECTION_FAILS			):
  with pytest.raises(requests.exceptions.ConnectionError			):
   requests.request("""GET"""					, """https://huggingface.co"""			)


def lowerCAmelCase_   (					):






 """simple docstring"""







 with offline(OfflineSimulationMode.HF_DATASETS_OFFLINE_SET_TO_1			):
  with pytest.raises(UpperCamelCase__			):
   http_head("""https://huggingface.co"""			)
