
"""simple docstring"""




import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
						from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
						from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
						from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


lowerCAmelCase__				=					DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

lowerCAmelCase__				=					'main'
# Default branch name
lowerCAmelCase__				=					'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
lowerCAmelCase__				=					'aaaaaaa'
# This commit does not exist, so we should 404.

lowerCAmelCase__				=					'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
lowerCAmelCase__				=					'4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'



@contextlib.contextmanager
def 							_lowerCamelCase			(			):
		print('''Welcome!'''     )
		yield
		print('''Bye!'''     )



@contextlib.contextmanager
def 							_lowerCamelCase			(			):
		print('''Bonjour!'''     )
		yield
		print('''Au revoir!'''     )





class 			snake_case  (				unittest.TestCase ):

							def _lowercase       (self     ):

									"""simple docstring"""







									assert transformers.__spec__ is not None
									assert importlib.util.find_spec('''transformers'''     ) is not None







class 			snake_case  (				unittest.TestCase ):

							@unittest.mock.patch('''sys.stdout'''							,							new_callable=io.StringIO     )
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									with ContextManagers([]     ):
											print('''Transformers are awesome!'''     )
									# The print statement adds a new line at the end of the output
									self.assertEqual(mock_stdout.getvalue()							,							'''Transformers are awesome!\n'''     )

							@unittest.mock.patch('''sys.stdout'''							,							new_callable=io.StringIO     )
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									with ContextManagers([context_en()]     ):
											print('''Transformers are awesome!'''     )
									# The output should be wrapped with an English welcome and goodbye
									self.assertEqual(mock_stdout.getvalue()							,							'''Welcome!\nTransformers are awesome!\nBye!\n'''     )

							@unittest.mock.patch('''sys.stdout'''							,							new_callable=io.StringIO     )
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									with ContextManagers([context_fr(), context_en()]     ):
											print('''Transformers are awesome!'''     )
									# The output should be wrapped with an English and French welcome and goodbye
									self.assertEqual(mock_stdout.getvalue()							,							'''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''     )

							@require_torch
							def _lowercase       (self     ):

									"""simple docstring"""







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''']     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''', '''next_sentence_label''']     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''start_positions''', '''end_positions''']     )





									class 			snake_case  (				__lowercase ):
																pass







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''']     )

							@require_tf
							def _lowercase       (self     ):

									"""simple docstring"""







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''']     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''', '''next_sentence_label''']     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''start_positions''', '''end_positions''']     )





									class 			snake_case  (				__lowercase ):
																pass







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							['''labels''']     )







							@require_flax
							def _lowercase       (self     ):

									"""simple docstring"""







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							[]     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							[]     )
									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							[]     )





									class 			snake_case  (				__lowercase ):
																pass







									self.assertEqual(find_labels(SCREAMING_SNAKE_CASE_     )							,							[]     )

"""simple docstring"""




import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 			snake_case  (				__lowercase							,      unittest.TestCase ):
							UpperCAmelCase__      =   None
							UpperCAmelCase__      =   BloomTokenizerFast
							UpperCAmelCase__      =   BloomTokenizerFast
							UpperCAmelCase__      =   True
							UpperCAmelCase__      =   False
							UpperCAmelCase__      =   '''tokenizer_file'''
							UpperCAmelCase__      =   {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}

							def _lowercase       (self     ):

									"""simple docstring"""







									super().setUp()
									SCREAMING_SNAKE_CASE_    			=			BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''     )
									tokenizer.save_pretrained(self.tmpdirname     )

							def _lowercase       (self							,							**SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									kwargs.update(self.special_tokens_map     )
									return BloomTokenizerFast.from_pretrained(self.tmpdirname							,							**SCREAMING_SNAKE_CASE_     )

							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.get_rust_tokenizer()

									SCREAMING_SNAKE_CASE_    			=			['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
									SCREAMING_SNAKE_CASE_    			=			[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

									SCREAMING_SNAKE_CASE_    			=			tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE_     )['''input_ids''']
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

									SCREAMING_SNAKE_CASE_    			=			tokenizer.batch_decode(SCREAMING_SNAKE_CASE_     )
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_=6     ):

									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
													SCREAMING_SNAKE_CASE_    			=			self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_							,							**SCREAMING_SNAKE_CASE_     )
													# tokenizer_r.pad_token = None # Hotfixing padding = None
													# Simple input
													SCREAMING_SNAKE_CASE_    			=			'''This is a simple input'''
													SCREAMING_SNAKE_CASE_    			=			['''This is a simple input 1''', '''This is a simple input 2''']
													SCREAMING_SNAKE_CASE_    			=			('''This is a simple input''', '''This is a pair''')
													SCREAMING_SNAKE_CASE_    			=			[
													    ('''This is a simple input 1''', '''This is a simple input 2'''),
													    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
													]

													# Simple input tests
													try:
															tokenizer_r.encode(SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_     )
															tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_     )

															tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_     )
															tokenizer_r.encode(SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_     )
															tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_     )
													except ValueError:
															self.fail('''Bloom Tokenizer should be able to deal with padding'''     )

													SCREAMING_SNAKE_CASE_    			=			None  # Hotfixing padding = None
													self.assertRaises(SCREAMING_SNAKE_CASE_							,							tokenizer_r.encode							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''     )

													# Simple input
													self.assertRaises(SCREAMING_SNAKE_CASE_							,							tokenizer_r.encode_plus							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''     )

													# Simple input
													self.assertRaises(
													    SCREAMING_SNAKE_CASE_							,							tokenizer_r.batch_encode_plus							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''							,							)

													# Pair input
													self.assertRaises(SCREAMING_SNAKE_CASE_							,							tokenizer_r.encode							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''     )

													# Pair input
													self.assertRaises(SCREAMING_SNAKE_CASE_							,							tokenizer_r.encode_plus							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''     )

													# Pair input
													self.assertRaises(
													    SCREAMING_SNAKE_CASE_							,							tokenizer_r.batch_encode_plus							,							SCREAMING_SNAKE_CASE_							,							max_length=SCREAMING_SNAKE_CASE_							,							padding='''max_length'''							,							)

							def _lowercase       (self     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.get_rust_tokenizer()
									SCREAMING_SNAKE_CASE_    			=			load_dataset('''xnli'''							,							'''all_languages'''							,							split='''test'''							,							streaming=SCREAMING_SNAKE_CASE_     )

									SCREAMING_SNAKE_CASE_    			=			next(iter(SCREAMING_SNAKE_CASE_     )     )['''premise''']  # pick up one data
									SCREAMING_SNAKE_CASE_    			=			list(sample_data.values()     )

									SCREAMING_SNAKE_CASE_    			=			list(map(tokenizer.encode							,							SCREAMING_SNAKE_CASE_     )     )
									SCREAMING_SNAKE_CASE_    			=			[tokenizer.decode(SCREAMING_SNAKE_CASE_							,							clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_     ) for x in output_tokens]
									self.assertListEqual(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self     ):

									"""simple docstring"""







									self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map     )							,							1     )
									self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()     )[0]     )							,							1     )