






"""simple docstring"""
import inspect
import unittest
import warnings

from transformers import DeiTConfig
from transformers.models.auto import get_values
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
    require_torch_gpu,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch
			from torch import nn

			from transformers import (
			    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
			    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    MODEL_MAPPING,
			    DeiTForImageClassification,
			    DeiTForImageClassificationWithTeacher,
			    DeiTForMaskedImageModeling,
			    DeiTModel,
			)
			from transformers.models.deit.modeling_deit import DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
			from PIL import Image

			from transformers import DeiTImageProcessor







class 			UpperCamelCase_       :





	def __init__(      self					: str	,	lowerCAmelCase_					: Any	,	lowerCAmelCase_					: List[str]=13	,	lowerCAmelCase_					: str=30	,	lowerCAmelCase_					: str=2	,	lowerCAmelCase_					: Tuple=3	,	lowerCAmelCase_					: str=True	,	lowerCAmelCase_					: Any=True	,	lowerCAmelCase_					: Tuple=32	,	lowerCAmelCase_					: Any=5	,	lowerCAmelCase_					: int=4	,	lowerCAmelCase_					: Any=37	,	lowerCAmelCase_					: Optional[int]="gelu"	,	lowerCAmelCase_					: Optional[int]=0.1	,	lowerCAmelCase_					: Dict=0.1	,	lowerCAmelCase_					: str=10	,	lowerCAmelCase_					: str=0.0_2	,	lowerCAmelCase_					: Any=3	,	lowerCAmelCase_					: Optional[Any]=None	,	lowerCAmelCase_					: List[str]=2	,	)						->			Optional[Any]:
				UpperCAmelCase_     : str			     =					parent
				UpperCAmelCase_     : str			     =					batch_size
				UpperCAmelCase_     : Optional[Any]			     =					image_size
				UpperCAmelCase_     : List[Any]			     =					patch_size
				UpperCAmelCase_     : Optional[Any]			     =					num_channels
				UpperCAmelCase_     : str			     =					is_training
				UpperCAmelCase_     : Tuple			     =					use_labels
				UpperCAmelCase_     : Optional[int]			     =					hidden_size
				UpperCAmelCase_     : str			     =					num_hidden_layers
				UpperCAmelCase_     : Optional[int]			     =					num_attention_heads
				UpperCAmelCase_     : List[Any]			     =					intermediate_size
				UpperCAmelCase_     : List[str]			     =					hidden_act
				UpperCAmelCase_     : Tuple			     =					hidden_dropout_prob
				UpperCAmelCase_     : str			     =					attention_probs_dropout_prob
				UpperCAmelCase_     : Any			     =					type_sequence_label_size
				UpperCAmelCase_     : List[Any]			     =					initializer_range
				UpperCAmelCase_     : Optional[Any]			     =					scope
				UpperCAmelCase_     : Optional[Any]			     =					encoder_stride

				# in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
				UpperCAmelCase_     : Optional[Any]			     =					(image_size // patch_size) ** 2
				UpperCAmelCase_     : Optional[Any]			     =					num_patches + 2





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			int:
				UpperCAmelCase_     : Optional[int]			     =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

				UpperCAmelCase_     : Optional[int]			     =					None
				if self.use_labels:
							UpperCAmelCase_     : List[Any]			     =					ids_tensor([self.batch_size]	,	self.type_sequence_label_size						)

				UpperCAmelCase_     : Any			     =					self.get_config()

				return config, pixel_values, labels





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			str:
				return DeiTConfig(
				    image_size=self.image_size	,	patch_size=self.patch_size	,	num_channels=self.num_channels	,	hidden_size=self.hidden_size	,	num_hidden_layers=self.num_hidden_layers	,	num_attention_heads=self.num_attention_heads	,	intermediate_size=self.intermediate_size	,	hidden_act=self.hidden_act	,	hidden_dropout_prob=self.hidden_dropout_prob	,	attention_probs_dropout_prob=self.attention_probs_dropout_prob	,	is_decoder=lowerCAmelCase_	,	initializer_range=self.initializer_range	,	encoder_stride=self.encoder_stride	,	)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: Optional[int]	,	lowerCAmelCase_					: List[str]						)						->			Union[str, Any]:
				UpperCAmelCase_     : str			     =					DeiTModel(config=lowerCAmelCase_						)
				model.to(lowerCAmelCase_						)
				model.eval()
				UpperCAmelCase_     : Union[str, Any]			     =					model(lowerCAmelCase_						)
				self.parent.assertEqual(result.last_hidden_state.shape	,	(self.batch_size, self.seq_length, self.hidden_size)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: Union[str, Any]	,	lowerCAmelCase_					: Union[str, Any]						)						->			Tuple:
				UpperCAmelCase_     : int			     =					DeiTForMaskedImageModeling(config=lowerCAmelCase_						)
				model.to(lowerCAmelCase_						)
				model.eval()
				UpperCAmelCase_     : str			     =					model(lowerCAmelCase_						)
				self.parent.assertEqual(
				    result.reconstruction.shape	,	(self.batch_size, self.num_channels, self.image_size, self.image_size)						)

				# test greyscale images
				UpperCAmelCase_     : Union[str, Any]			     =					1
				UpperCAmelCase_     : List[str]			     =					DeiTForMaskedImageModeling(lowerCAmelCase_						)
				model.to(lowerCAmelCase_						)
				model.eval()

				UpperCAmelCase_     : Tuple			     =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
				UpperCAmelCase_     : Tuple			     =					model(lowerCAmelCase_						)
				self.parent.assertEqual(result.reconstruction.shape	,	(self.batch_size, 1, self.image_size, self.image_size)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: Optional[Any]	,	lowerCAmelCase_					: Optional[int]	,	lowerCAmelCase_					: Dict						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					self.type_sequence_label_size
				UpperCAmelCase_     : Dict			     =					DeiTForImageClassification(lowerCAmelCase_						)
				model.to(lowerCAmelCase_						)
				model.eval()
				UpperCAmelCase_     : str			     =					model(lowerCAmelCase_	,	labels=lowerCAmelCase_						)
				self.parent.assertEqual(result.logits.shape	,	(self.batch_size, self.type_sequence_label_size)						)

				# test greyscale images
				UpperCAmelCase_     : Any			     =					1
				UpperCAmelCase_     : Dict			     =					DeiTForImageClassification(lowerCAmelCase_						)
				model.to(lowerCAmelCase_						)
				model.eval()

				UpperCAmelCase_     : List[Any]			     =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]						)
				UpperCAmelCase_     : List[str]			     =					model(lowerCAmelCase_	,	labels=lowerCAmelCase_						)
				self.parent.assertEqual(result.logits.shape	,	(self.batch_size, self.type_sequence_label_size)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			List[Any]:
				UpperCAmelCase_     : Any			     =					self.prepare_config_and_inputs()
				(
				    (
				    UpperCAmelCase_
				)     ,	(
				    UpperCAmelCase_
				)     ,	(
				    UpperCAmelCase_
				)     ,	
				)     : Optional[int]			     =					config_and_inputs
				UpperCAmelCase_     : Optional[Any]			     =					{"pixel_values": pixel_values}
				return config, inputs_dict







@require_torch
class 			UpperCamelCase_       (__A				,						__A				,						unittest.TestCase     ):
	__magic_name__	      =				(
	    (
	        DeiTModel,
	        DeiTForImageClassification,
	        DeiTForImageClassificationWithTeacher,
	        DeiTForMaskedImageModeling,
	    )
	    if is_torch_available()
	    else ()
	)
	__magic_name__	      =				(
	    {
	        '''feature-extraction''': DeiTModel,
	        '''image-classification''': (DeiTForImageClassification, DeiTForImageClassificationWithTeacher),
	    }
	    if is_torch_available()
	    else {}
	)

	__magic_name__	      =				False
	__magic_name__	      =				False
	__magic_name__	      =				False





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     : Tuple			     =					DeiTModelTester(self						)
				UpperCAmelCase_     : Tuple			     =					ConfigTester(self	,	config_class=lowerCAmelCase_	,	has_text_modality=lowerCAmelCase_	,	hidden_size=37						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Union[str, Any]:
				self.config_tester.run_common_tests()





	@unittest.skip(reason="DeiT does not use inputs_embeds"						)
	def      _SCREAMING_SNAKE_CASE      (      self					: int						)						->			Optional[int]:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Union[str, Any]:
				UpperCAmelCase_     ,	UpperCAmelCase_     : Optional[int]			     =					self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
							UpperCAmelCase_     : Dict			     =					model_class(lowerCAmelCase_						)
							self.assertIsInstance(model.get_input_embeddings()	,	(nn.Module)						)
							UpperCAmelCase_     : int			     =					model.get_output_embeddings()
							self.assertTrue(x is None or isinstance(lowerCAmelCase_	,	nn.Linear						)						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			List[str]:
				UpperCAmelCase_     ,	UpperCAmelCase_     : Optional[Any]			     =					self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
							UpperCAmelCase_     : List[Any]			     =					model_class(lowerCAmelCase_						)
							UpperCAmelCase_     : Any			     =					inspect.signature(model.forward						)
							# signature.parameters is an OrderedDict => so arg_names order is deterministic
							UpperCAmelCase_     : Union[str, Any]			     =					[*signature.parameters.keys()]

							UpperCAmelCase_     : str			     =					["pixel_values"]
							self.assertListEqual(arg_names[:1]	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				UpperCAmelCase_     : Tuple			     =					self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Union[str, Any]:
				UpperCAmelCase_     : Tuple			     =					self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_image_modeling(*lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			Optional[int]:
				UpperCAmelCase_     : List[Any]			     =					self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict	,	lowerCAmelCase_					: Tuple	,	lowerCAmelCase_					: Any	,	lowerCAmelCase_					: Union[str, Any]=False						)						->			List[Any]:
				UpperCAmelCase_     : Optional[Any]			     =					super()._prepare_for_class(lowerCAmelCase_	,	lowerCAmelCase_	,	return_labels=lowerCAmelCase_						)

				if return_labels:
							if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
										del inputs_dict["labels"]

				return inputs_dict





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			Tuple:
				if not self.model_tester.is_training:
							return

				UpperCAmelCase_     ,	UpperCAmelCase_     : Dict			     =					self.model_tester.prepare_config_and_inputs_for_common()
				UpperCAmelCase_     : Union[str, Any]			     =					True

				for model_class in self.all_model_classes:
							# DeiTForImageClassificationWithTeacher supports inference-only
							if (
							    model_class in get_values(lowerCAmelCase_						)
							    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
							):
										continue
							UpperCAmelCase_     : List[str]			     =					model_class(lowerCAmelCase_						)
							model.to(lowerCAmelCase_						)
							model.train()
							UpperCAmelCase_     : Union[str, Any]			     =					self._prepare_for_class(lowerCAmelCase_	,	lowerCAmelCase_	,	return_labels=lowerCAmelCase_						)
							UpperCAmelCase_     : Any			     =					model(**lowerCAmelCase_						).loss
							loss.backward()





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Optional[Any]:
				UpperCAmelCase_     ,	UpperCAmelCase_     : int			     =					self.model_tester.prepare_config_and_inputs_for_common()
				if not self.model_tester.is_training:
							return

				UpperCAmelCase_     : Union[str, Any]			     =					False
				UpperCAmelCase_     : Union[str, Any]			     =					True

				for model_class in self.all_model_classes:
							if model_class in get_values(lowerCAmelCase_						) or not model_class.supports_gradient_checkpointing:
										continue
							# DeiTForImageClassificationWithTeacher supports inference-only
							if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
										continue
							UpperCAmelCase_     : List[str]			     =					model_class(lowerCAmelCase_						)
							model.gradient_checkpointing_enable()
							model.to(lowerCAmelCase_						)
							model.train()
							UpperCAmelCase_     : Optional[Any]			     =					self._prepare_for_class(lowerCAmelCase_	,	lowerCAmelCase_	,	return_labels=lowerCAmelCase_						)
							UpperCAmelCase_     : int			     =					model(**lowerCAmelCase_						).loss
							loss.backward()





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			List[str]:
				UpperCAmelCase_     ,	UpperCAmelCase_     : List[str]			     =					self.model_tester.prepare_config_and_inputs_for_common()

				UpperCAmelCase_     : str			     =					[
				    {"title": "multi_label_classification", "num_labels": 2, "dtype": torch.float},
				    {"title": "single_label_classification", "num_labels": 1, "dtype": torch.long},
				    {"title": "regression", "num_labels": 1, "dtype": torch.float},
				]

				for model_class in self.all_model_classes:
							if (
							    model_class
							    not in [
							        *get_values(lowerCAmelCase_						),
							        *get_values(lowerCAmelCase_						),
							    ]
							    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
							):
										continue

							for problem_type in problem_types:
										with self.subTest(msg=f"""Testing {model_class} with {problem_type['title']}"""						):
													UpperCAmelCase_     : List[Any]			     =					problem_type["title"]
													UpperCAmelCase_     : List[str]			     =					problem_type["num_labels"]

													UpperCAmelCase_     : Any			     =					model_class(lowerCAmelCase_						)
													model.to(lowerCAmelCase_						)
													model.train()

													UpperCAmelCase_     : Optional[Any]			     =					self._prepare_for_class(lowerCAmelCase_	,	lowerCAmelCase_	,	return_labels=lowerCAmelCase_						)

													if problem_type["num_labels"] > 1:
																UpperCAmelCase_     : Union[str, Any]			     =					inputs["labels"].unsqueeze(1						).repeat(1	,	problem_type["num_labels"]						)

													UpperCAmelCase_     : Optional[int]			     =					inputs["labels"].to(problem_type["dtype"]						)

													# This tests that we do not trigger the warning form PyTorch "Using a target size that is different
													# to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
													# they have the same size." which is a symptom something in wrong for the regression problem.
													# See https://github.com/huggingface/transformers/issues/11780
													with warnings.catch_warnings(record=lowerCAmelCase_						) as warning_list:
																UpperCAmelCase_     : Any			     =					model(**lowerCAmelCase_						).loss
													for w in warning_list:
																if "Using a target size that is different to the input size" in str(w.message						):
																			raise ValueError(
																			    f"""Something is going wrong in the regression problem: intercepted {w.message}"""						)

													loss.backward()





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Any:
				for model_name in DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							UpperCAmelCase_     : Optional[int]			     =					DeiTModel.from_pretrained(lowerCAmelCase_						)
							self.assertIsNotNone(lowerCAmelCase_						)





def 				snake_case      (   ):
			UpperCAmelCase_     : Dict			     =					Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"						)
			return image







@require_torch
@require_vision
class 			UpperCamelCase_       (unittest.TestCase     ):





	@cached_property
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]						)						->			str:
				return (
				    DeiTImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224"						)
				    if is_vision_available()
				    else None
				)





	@slow
	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[Any]:
				UpperCAmelCase_     : Any			     =					DeiTForImageClassificationWithTeacher.from_pretrained("facebook/deit-base-distilled-patch16-224"						).to(
				    lowerCAmelCase_						)

				UpperCAmelCase_     : Union[str, Any]			     =					self.default_image_processor
				UpperCAmelCase_     : Optional[int]			     =					prepare_img()
				UpperCAmelCase_     : Any			     =					image_processor(images=lowerCAmelCase_	,	return_tensors="pt"						).to(lowerCAmelCase_						)

				# forward pass
				with torch.no_grad():
							UpperCAmelCase_     : Dict			     =					model(**lowerCAmelCase_						)

				# verify the logits
				UpperCAmelCase_     : str			     =					torch.Size((1, 1_000)						)
				self.assertEqual(outputs.logits.shape	,	lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[int]			     =					torch.tensor([-1.0_2_6_6, 0.1_9_1_2, -1.2_8_6_1]						).to(lowerCAmelCase_						)

				self.assertTrue(torch.allclose(outputs.logits[0, :3]	,	lowerCAmelCase_	,	atol=1e-4						)						)





	@slow
	@require_accelerate
	@require_torch_gpu
	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]						)						->			Dict:
				UpperCAmelCase_     : Union[str, Any]			     =					DeiTModel.from_pretrained(
				    "facebook/deit-base-distilled-patch16-224"	,	torch_dtype=torch.floataa	,	device_map="auto"						)
				UpperCAmelCase_     : Optional[int]			     =					self.default_image_processor

				UpperCAmelCase_     : Union[str, Any]			     =					prepare_img()
				UpperCAmelCase_     : int			     =					image_processor(images=lowerCAmelCase_	,	return_tensors="pt"						)
				UpperCAmelCase_     : Optional[int]			     =					inputs.pixel_values.to(lowerCAmelCase_						)

				# forward pass to make sure inference works in fp16
				with torch.no_grad():
							UpperCAmelCase_     : int			     =					model(lowerCAmelCase_						)











"""simple docstring"""
# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


lowerCamelCase_					=      get_logger(__name__)







class 			UpperCamelCase_       :
	__magic_name__	      =				'''dummy_data'''
	__magic_name__	      =				'''datasets'''
	__magic_name__	      =				False





	def __init__(      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Union[Version, str]	,	lowerCAmelCase_					: Optional[str] = None	,	lowerCAmelCase_					: bool = False	,	lowerCAmelCase_					: bool = True	,	lowerCAmelCase_					: Optional[List[Callable]] = None	,	)						->			Tuple:
				UpperCAmelCase_     : Optional[int]			     =					0
				UpperCAmelCase_     : int			     =					dataset_name
				UpperCAmelCase_     : Optional[int]			     =					cache_dir
				UpperCAmelCase_     : Tuple			     =					use_local_dummy_data
				UpperCAmelCase_     : int			     =					config
				# download_callbacks take a single url as input
				UpperCAmelCase_     : List[Callable]			     =					download_callbacks or []
				# if False, it doesn't load existing files and it returns the paths of the dummy files relative
				# to the dummy_data zip file root
				UpperCAmelCase_     : Optional[Any]			     =					load_existing_dummy_data

				# TODO(PVP, QL) might need to make this more general
				UpperCAmelCase_     : Dict			     =					str(lowerCAmelCase_						)
				# to be downloaded
				UpperCAmelCase_     : Optional[Any]			     =					None
				UpperCAmelCase_     : int			     =					None





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			str:
				if self._dummy_file is None:
							UpperCAmelCase_     : List[str]			     =					self.download_dummy_data()
				return self._dummy_file





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			int:
				if self.config is not None:
							# structure is dummy / config_name / version_name
							return os.path.join("dummy"	,	self.config.name	,	self.version_name						)
				# structure is dummy / version_name
				return os.path.join("dummy"	,	self.version_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				return os.path.join(self.dummy_data_folder	,	"dummy_data.zip"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Tuple:
				UpperCAmelCase_     : int			     =					(
				    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
				)

				UpperCAmelCase_     : Union[str, Any]			     =					cached_path(
				    lowerCAmelCase_	,	cache_dir=self.cache_dir	,	extract_compressed_file=lowerCAmelCase_	,	force_extract=lowerCAmelCase_						)

				return os.path.join(lowerCAmelCase_	,	self.dummy_file_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			int:
				return os.path.join(self.datasets_scripts_dir	,	self.dataset_name	,	self.dummy_zip_file						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[Any]:
				if self._bucket_url is None:
							UpperCAmelCase_     : Union[str, Any]			     =					hf_github_url(self.dataset_name	,	self.dummy_zip_file.replace(os.sep	,	"/"						)						)
				return self._bucket_url





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Optional[Any]:
				# return full path if its a dir
				if os.path.isdir(self.dummy_file						):
							return self.dummy_file
				# else cut off path to file -> example `xsum`.
				return "/".join(self.dummy_file.replace(os.sep	,	"/"						).split("/"						)[:-1]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: List[str]	,	*lowerCAmelCase_					: List[Any]						)						->			Optional[int]:
				if self.load_existing_dummy_data:
							# dummy data is downloaded and tested
							UpperCAmelCase_     : Dict			     =					self.dummy_file
				else:
							# dummy data cannot be downloaded and only the path to dummy file is returned
							UpperCAmelCase_     : Optional[int]			     =					self.dummy_file_name

				# special case when data_url is a dict
				if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							return self.create_dummy_data_dict(lowerCAmelCase_	,	lowerCAmelCase_						)
				elif isinstance(lowerCAmelCase_	,	(list, tuple)						):
							return self.create_dummy_data_list(lowerCAmelCase_	,	lowerCAmelCase_						)
				else:
							return self.create_dummy_data_single(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]	,	lowerCAmelCase_					: int	,	*lowerCAmelCase_					: Union[str, Any]						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: Tuple						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: Union[str, Any]	,	*lowerCAmelCase_					: Tuple	,	**lowerCAmelCase_					: Tuple						)						->			Union[str, Any]:
				return path





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			List[str]:
				return {}





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: List[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Dict			     =					{}
				for key, single_urls in data_url.items():
							for download_callback in self.download_callbacks:
										if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
													for single_url in single_urls:
																download_callback(lowerCAmelCase_						)
										else:
													UpperCAmelCase_     : Tuple			     =					single_urls
													download_callback(lowerCAmelCase_						)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
							if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
										UpperCAmelCase_     : List[str]			     =					[os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						) for x in single_urls]
							else:
										UpperCAmelCase_     : Optional[int]			     =					single_urls
										UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						)
							UpperCAmelCase_     : int			     =					value

				# make sure that values are unique
				if all(isinstance(lowerCAmelCase_	,	lowerCAmelCase_						) for i in dummy_data_dict.values()						) and len(set(dummy_data_dict.values()						)						) < len(
				    dummy_data_dict.values()						):
							# append key to value to make its name unique
							UpperCAmelCase_     : List[str]			     =					{key: value + key for key, value in dummy_data_dict.items()}

				return dummy_data_dict





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: List[Any]	,	lowerCAmelCase_					: Optional[int]						)						->			Dict:
				UpperCAmelCase_     : str			     =					[]
				# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
				UpperCAmelCase_     : int			     =					all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}"	,	lowerCAmelCase_						)						) for url in data_url						)
				UpperCAmelCase_     : Union[str, Any]			     =					all(
				    url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"						) for url in data_url						)
				if data_url and (is_tf_records or is_pubmed_records):
							UpperCAmelCase_     : Tuple			     =					[data_url[0]] * len(lowerCAmelCase_						)
				for single_url in data_url:
							for download_callback in self.download_callbacks:
										download_callback(lowerCAmelCase_						)
							# we force the name of each key to be the last file / folder name of the url path
							# if the url has arguments, we need to encode them with urllib.parse.quote_plus
							UpperCAmelCase_     : Dict			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(single_url.split("/"						)[-1]						)						)
							dummy_data_list.append(lowerCAmelCase_						)
				return dummy_data_list





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: Optional[Any]	,	lowerCAmelCase_					: str						)						->			Optional[int]:
				for download_callback in self.download_callbacks:
							download_callback(lowerCAmelCase_						)
				# we force the name of each key to be the last file / folder name of the url path
				# if the url has arguments, we need to encode them with urllib.parse.quote_plus
				UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(data_url.split("/"						)[-1]						)						)
				if os.path.exists(lowerCAmelCase_						) or not self.load_existing_dummy_data:
							return value
				else:
							# Backward compatibility, maybe deprecate at one point.
							# For many datasets with single url calls to dl_manager.download_and_extract,
							# the dummy_data.zip file is actually the zipped downloaded file
							# while now we expected the dummy_data.zip file to be a directory containing
							# the downloaded file.
							return path_to_dummy_data





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			int:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: Dict						)						->			Optional[Any]:
				def _iter_archive_members(lowerCAmelCase_					: Dict						):
							# this preserves the order of the members inside the ZIP archive
							UpperCAmelCase_     : str			     =					Path(self.dummy_file						).parent
							UpperCAmelCase_     : Optional[Any]			     =					path.relative_to(lowerCAmelCase_						)
							with ZipFile(self.local_path_to_dummy_data						) as zip_file:
										UpperCAmelCase_     : str			     =					zip_file.namelist()
							for member in members:
										if member.startswith(relative_path.as_posix()						):
													yield dummy_parent_path.joinpath(lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					Path(lowerCAmelCase_						)
				UpperCAmelCase_     : List[Any]			     =					_iter_archive_members(lowerCAmelCase_						) if self.use_local_dummy_data else path.rglob("*"						)
				for file_path in file_paths:
							if file_path.is_file() and not file_path.name.startswith((".", "__")						):
										yield file_path.relative_to(lowerCAmelCase_						).as_posix(), file_path.open("rb"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: Tuple						)						->			str:
				if not isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							UpperCAmelCase_     : str			     =					[paths]
				for path in paths:
							if os.path.isfile(lowerCAmelCase_						):
										if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
													return
										yield path
							else:
										for dirpath, dirnames, filenames in os.walk(lowerCAmelCase_						):
													if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
																continue
													dirnames.sort()
													for filename in sorted(lowerCAmelCase_						):
																if filename.startswith((".", "__")						):
																			continue
																yield os.path.join(lowerCAmelCase_	,	lowerCAmelCase_						)



