





"""simple docstring"""






import collections
import gzip
import os
import urllib

import numpy
from tensorflow.python.framework import dtypes, random_seed
from tensorflow.python.platform import gfile
from tensorflow.python.util.deprecation import deprecated

__UpperCAmelCase						=			collections.namedtuple('''_Datasets''', ['''train''', '''validation''', '''test'''])

# CVDF mirror of http://yann.lecun.com/exdb/mnist/
__UpperCAmelCase						=			'''https://storage.googleapis.com/cvdf-datasets/mnist/'''






def      lowercase__ (					lowerCAmelCase__   :			List[str]			)    ->			Tuple:



			'''simple docstring'''
			a__  :	Union[str, Any]		 =			numpy.dtype(numpy.uintaa			).newbyteorder(">"			)
			return numpy.frombuffer(bytestream.read(4			)							,			dtype=_UpperCamelCase			)[0]






@deprecated(_UpperCamelCase							,			"Please use tf.data to implement this functionality."			)
def      lowercase__ (					lowerCAmelCase__   :			Any			)    ->			Dict:



			'''simple docstring'''
			print("Extracting"							,			f.name			)
			with gzip.GzipFile(fileobj=_UpperCamelCase			) as bytestream:
						a__  :	Any		 =			_readaa(_UpperCamelCase			)
						if magic != 2_0_5_1:
									raise ValueError(
									    "Invalid magic number %d in MNIST image file: %s" % (magic, f.name)			)
						a__  :	str		 =			_readaa(_UpperCamelCase			)
						a__  :	Optional[Any]		 =			_readaa(_UpperCamelCase			)
						a__  :	Tuple		 =			_readaa(_UpperCamelCase			)
						a__  :	Optional[Any]		 =			bytestream.read(rows * cols * num_images			)
						a__  :	int		 =			numpy.frombuffer(_UpperCamelCase							,			dtype=numpy.uinta			)
						a__  :	Union[str, Any]		 =			data.reshape(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			1			)
						return data






@deprecated(_UpperCamelCase							,			"Please use tf.one_hot on tensors."			)
def      lowercase__ (					lowerCAmelCase__   :			Union[str, Any]							,			lowerCAmelCase__   :			List[str]			)    ->			Optional[int]:



			'''simple docstring'''
			a__  :	Optional[Any]		 =			labels_dense.shape[0]
			a__  :	List[str]		 =			numpy.arange(_UpperCamelCase			) * num_classes
			a__  :	int		 =			numpy.zeros((num_labels, num_classes)			)
			a__  :	Optional[int]		 =			1
			return labels_one_hot






@deprecated(_UpperCamelCase							,			"Please use tf.data to implement this functionality."			)
def      lowercase__ (					lowerCAmelCase__   :			Optional[int]							,			lowerCAmelCase__   :			Optional[int]=False							,			lowerCAmelCase__   :			Dict=1_0			)    ->			List[Any]:



			'''simple docstring'''
			print("Extracting"							,			f.name			)
			with gzip.GzipFile(fileobj=_UpperCamelCase			) as bytestream:
						a__  :	List[Any]		 =			_readaa(_UpperCamelCase			)
						if magic != 2_0_4_9:
									raise ValueError(
									    "Invalid magic number %d in MNIST label file: %s" % (magic, f.name)			)
						a__  :	str		 =			_readaa(_UpperCamelCase			)
						a__  :	str		 =			bytestream.read(_UpperCamelCase			)
						a__  :	Tuple		 =			numpy.frombuffer(_UpperCamelCase							,			dtype=numpy.uinta			)
						if one_hot:
									return _dense_to_one_hot(_UpperCamelCase							,			_UpperCamelCase			)
						return labels



class 					__UpperCAmelCase							:
				@deprecated(
				    a_	, "Please use alternatives such as official/mnist/_DataSet.py"
				    " from tensorflow/models."	, )
				def __init__(		self    :			Tuple	, a_    :			Union[str, Any]	, a_    :			Any	, a_    :			Optional[Any]=False	, a_    :			Dict=False	, a_    :			List[str]=dtypes.floataa	, a_    :			Optional[Any]=True	, a_    :			str=None	, )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	str		 =			random_seed.get_seed(a_    )
							# If op level seed is not set, use whatever graph level seed is returned
							numpy.random.seed(seeda if seed is None else seeda    )
							a__  :	Optional[int]		 =			dtypes.as_dtype(a_    ).base_dtype
							if dtype not in (dtypes.uinta, dtypes.floataa):
										raise TypeError("Invalid image dtype %r, expected uint8 or float32" % dtype    )
							if fake_data:
										a__  :	int		 =			1_00_00
										a__  :	List[Any]		 =			one_hot
							else:
										assert (
										    images.shape[0] == labels.shape[0]
										), F"images.shape: {images.shape} labels.shape: {labels.shape}"
										a__  :	List[str]		 =			images.shape[0]

										# Convert shape from [num examples, rows, columns, depth]
										# to [num examples, rows*columns] (assuming depth == 1)
										if reshape:
													assert images.shape[3] == 1
													a__  :	int		 =			images.reshape(
													    images.shape[0]	, images.shape[1] * images.shape[2]    )
										if dtype == dtypes.floataa:
													# Convert from [0, 255] -> [0.0, 1.0].
													a__  :	Any		 =			images.astype(numpy.floataa    )
													a__  :	List[str]		 =			numpy.multiply(a_	, 1.0 / 255.0    )
							a__  :	Optional[int]		 =			images
							a__  :	Optional[int]		 =			labels
							a__  :	Union[str, Any]		 =			0
							a__  :	Union[str, Any]		 =			0
				@property
				def       UpperCAmelCase      (		self    :			str    )	->	str:



							'''simple docstring'''



							return self._images
				@property
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Dict:



							'''simple docstring'''



							return self._labels
				@property
				def       UpperCAmelCase      (		self    :			List[str]    )	->	List[Any]:



							'''simple docstring'''



							return self._num_examples
				@property
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[int]:



							'''simple docstring'''



							return self._epochs_completed






				def       UpperCAmelCase      (		self    :			Optional[Any]	, a_    :			Dict	, a_    :			Tuple=False	, a_    :			Union[str, Any]=True    )	->	Tuple:



							'''simple docstring'''



							if fake_data:
										a__  :	Union[str, Any]		 =			[1] * 7_84
										a__  :	Union[str, Any]		 =			[1] + [0] * 9 if self.one_hot else 0
										return (
										    [fake_image for _ in range(a_    )],
										    [fake_label for _ in range(a_    )],
										)
							a__  :	List[str]		 =			self._index_in_epoch
							# Shuffle for the first epoch
							if self._epochs_completed == 0 and start == 0 and shuffle:
										a__  :	Optional[Any]		 =			numpy.arange(self._num_examples    )
										numpy.random.shuffle(a_    )
										a__  :	str		 =			self.images[perma]
										a__  :	Dict		 =			self.labels[perma]
							# Go to the next epoch
							if start + batch_size > self._num_examples:
										# Finished epoch
										self._epochs_completed += 1
										# Get the rest examples in this epoch
										a__  :	List[Any]		 =			self._num_examples - start
										a__  :	Optional[Any]		 =			self._images[start : self._num_examples]
										a__  :	int		 =			self._labels[start : self._num_examples]
										# Shuffle the data
										if shuffle:
													a__  :	Any		 =			numpy.arange(self._num_examples    )
													numpy.random.shuffle(a_    )
													a__  :	Any		 =			self.images[perm]
													a__  :	Optional[Any]		 =			self.labels[perm]
										# Start next epoch
										a__  :	Optional[int]		 =			0
										a__  :	Union[str, Any]		 =			batch_size - rest_num_examples
										a__  :	str		 =			self._index_in_epoch
										a__  :	str		 =			self._images[start:end]
										a__  :	Union[str, Any]		 =			self._labels[start:end]
										return (
										    numpy.concatenate((images_rest_part, images_new_part)	, axis=0    ),
										    numpy.concatenate((labels_rest_part, labels_new_part)	, axis=0    ),
										)
							else:
										self._index_in_epoch += batch_size
										a__  :	List[Any]		 =			self._index_in_epoch
										return self._images[start:end], self._labels[start:end]






@deprecated(_UpperCamelCase							,			"Please write your own downloading logic."			)
def      lowercase__ (					lowerCAmelCase__   :			int							,			lowerCAmelCase__   :			Tuple							,			lowerCAmelCase__   :			List[str]			)    ->			Any:



			'''simple docstring'''




			if not gfile.Exists(_UpperCamelCase			):
						gfile.MakeDirs(_UpperCamelCase			)
			a__  :	Optional[Any]		 =			os.path.join(_UpperCamelCase							,			_UpperCamelCase			)
			if not gfile.Exists(_UpperCamelCase			):
						urllib.request.urlretrieve(_UpperCamelCase							,			_UpperCamelCase			)  # noqa: S310
						with gfile.GFile(_UpperCamelCase			) as f:
									a__  :	Optional[Any]		 =			f.size()
						print("Successfully downloaded"							,			_UpperCamelCase							,			_UpperCamelCase							,			"bytes."			)
			return filepath






@deprecated(
    _UpperCamelCase							,			"Please use alternatives such as:" " tensorflow_datasets.load('mnist')"			)
def      lowercase__ (					lowerCAmelCase__   :			Union[str, Any]							,			lowerCAmelCase__   :			int=False							,			lowerCAmelCase__   :			Dict=False							,			lowerCAmelCase__   :			Optional[Any]=dtypes.floataa							,			lowerCAmelCase__   :			Optional[Any]=True							,			lowerCAmelCase__   :			Any=5_0_0_0							,			lowerCAmelCase__   :			List[Any]=None							,			lowerCAmelCase__   :			Tuple=DEFAULT_SOURCE_URL							,			)    ->			str:



			'''simple docstring'''




			if fake_data:

						def fake():
									return _DataSet(
									    []							,			[]							,			fake_data=_UpperCamelCase							,			one_hot=_UpperCamelCase							,			dtype=_UpperCamelCase							,			seed=_UpperCamelCase			)

						a__  :	Tuple		 =			fake()
						a__  :	Any		 =			fake()
						a__  :	List[str]		 =			fake()
						return _Datasets(train=_UpperCamelCase							,			validation=_UpperCamelCase							,			test=_UpperCamelCase			)

			if not source_url:  # empty string check
						a__  :	Any		 =			DEFAULT_SOURCE_URL

			a__  :	Tuple		 =			"train-images-idx3-ubyte.gz"
			a__  :	str		 =			"train-labels-idx1-ubyte.gz"
			a__  :	List[str]		 =			"t10k-images-idx3-ubyte.gz"
			a__  :	Union[str, Any]		 =			"t10k-labels-idx1-ubyte.gz"

			a__  :	List[str]		 =			_maybe_download(
			    _UpperCamelCase							,			_UpperCamelCase							,			source_url + train_images_file			)
			with gfile.Open(_UpperCamelCase							,			"rb"			) as f:
						a__  :	Tuple		 =			_extract_images(_UpperCamelCase			)

			a__  :	Union[str, Any]		 =			_maybe_download(
			    _UpperCamelCase							,			_UpperCamelCase							,			source_url + train_labels_file			)
			with gfile.Open(_UpperCamelCase							,			"rb"			) as f:
						a__  :	List[str]		 =			_extract_labels(_UpperCamelCase							,			one_hot=_UpperCamelCase			)

			a__  :	Any		 =			_maybe_download(
			    _UpperCamelCase							,			_UpperCamelCase							,			source_url + test_images_file			)
			with gfile.Open(_UpperCamelCase							,			"rb"			) as f:
						a__  :	Tuple		 =			_extract_images(_UpperCamelCase			)

			a__  :	Dict		 =			_maybe_download(
			    _UpperCamelCase							,			_UpperCamelCase							,			source_url + test_labels_file			)
			with gfile.Open(_UpperCamelCase							,			"rb"			) as f:
						a__  :	Dict		 =			_extract_labels(_UpperCamelCase							,			one_hot=_UpperCamelCase			)

			if not 0 <= validation_size <= len(_UpperCamelCase			):
						a__  :	Dict		 =			(
						    "Validation size should be between 0 and "
						    F"{len(_UpperCamelCase			)}. Received: {validation_size}."
						)
						raise ValueError(_UpperCamelCase			)

			a__  :	int		 =			train_images[:validation_size]
			a__  :	Union[str, Any]		 =			train_labels[:validation_size]
			a__  :	str		 =			train_images[validation_size:]
			a__  :	Dict		 =			train_labels[validation_size:]

			a__  :	Optional[Any]		 =			{"dtype": dtype, "reshape": reshape, "seed": seed}

			a__  :	Any		 =			_DataSet(_UpperCamelCase							,			_UpperCamelCase							,			**_UpperCamelCase			)
			a__  :	Dict		 =			_DataSet(_UpperCamelCase							,			_UpperCamelCase							,			**_UpperCamelCase			)
			a__  :	Any		 =			_DataSet(_UpperCamelCase							,			_UpperCamelCase							,			**_UpperCamelCase			)

			return _Datasets(train=_UpperCamelCase							,			validation=_UpperCamelCase							,			test=_UpperCamelCase			)







'''simple docstring'''
import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
 from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
 from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
 from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


UpperCamelCase__			=      DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

UpperCamelCase__			=      'main'
# Default branch name
UpperCamelCase__			=      'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
UpperCamelCase__			=      'aaaaaaa'
# This commit does not exist, so we should 404.

UpperCamelCase__			=      'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
UpperCamelCase__			=      '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Welcome!"				)
   yield
   print("Bye!"				)




@contextlib.contextmanager
def 		__SCREAMING_SNAKE_CASE       (	):


   """simple docstring"""




   print("Bonjour!"				)
   yield
   print("Au revoir!"				)


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       assert transformers.__spec__ is not None
       assert importlib.util.find_spec("transformers"      ) is not None


class 							_UpperCAmelCase    (		unittest.TestCase      ):




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							str	,  a	:							List[str]      ):






       '''simple docstring'''






       with ContextManagers([]      ):
          print("Transformers are awesome!"      )
       # The print statement adds a new line at the end of the output
       self.assertEqual(mock_stdout.getvalue()	,  "Transformers are awesome!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							List[str]	,  a	:							Optional[int]      ):






       '''simple docstring'''






       with ContextManagers([context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Welcome!\nTransformers are awesome!\nBye!\n"      )




    @unittest.mock.patch("sys.stdout"	,  new_callable=io.StringIO      )
    def 					lowerCAmelCase__					(      self	:							Tuple	,  a	:							List[Any]      ):






       '''simple docstring'''






       with ContextManagers([context_fr(), context_en()]      ):
          print("Transformers are awesome!"      )
       # The output should be wrapped with an English and French welcome and goodbye
       self.assertEqual(mock_stdout.getvalue()	,  "Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"      )




    @require_torch
    def 					lowerCAmelCase__					(      self	:							Tuple      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )




    @require_tf
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  ["labels"]      )
       self.assertEqual(find_labels(a      )	,  ["labels", "next_sentence_label"]      )
       self.assertEqual(find_labels(a      )	,  ["start_positions", "end_positions"]      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  ["labels"]      )





    @require_flax
    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )
       self.assertEqual(find_labels(a      )	,  []      )


       class 							_UpperCAmelCase    (		snake_case      ):
           pass


       self.assertEqual(find_labels(a      )	,  []      )




