



"""simple docstring"""
import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     a		(     __A		,					unittest.TestCase				):
  UpperCamelCase		: str     =    RobertaTokenizer
  UpperCamelCase		: Optional[int]     =    RobertaTokenizerFast
  UpperCamelCase		: Any     =    True
  UpperCamelCase		: Optional[Any]     =    {'cls_token': '<s>'}



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    Tuple:


     '''simple docstring'''




     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=[
         "l",
         "o",
         "w",
         "e",
         "r",
         "s",
         "t",
         "i",
         "d",
         "n",
         "\u0120",
         "\u0120l",
         "\u0120n",
         "\u0120lo",
         "\u0120low",
         "er",
         "\u0120lowest",
         "\u0120newer",
         "\u0120wider",
         "<unk>",
     ]
     SCREAMING_SNAKE_CASE_:       str   	=dict(zip(lowerCAmelCase_				,						range(len(lowerCAmelCase_						)						)						)						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
     SCREAMING_SNAKE_CASE_:       Any   	={"unk_token": "<unk>"}

     SCREAMING_SNAKE_CASE_:       int   	=os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES["""vocab_file"""]						)
     SCREAMING_SNAKE_CASE_:       Tuple   	=os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES["""merges_file"""]						)
     with open(self.vocab_file				,						"""w"""				,						encoding="""utf-8"""						) as fp:
        fp.write(json.dumps(lowerCAmelCase_						) + """\n"""						)
     with open(self.merges_file				,						"""w"""				,						encoding="""utf-8"""						) as fp:
        fp.write("""\n""".join(lowerCAmelCase_						)						)



  def 		lowerCamelCase__				(  self       :    Tuple				,						**lowerCAmelCase       :    int						)   ->    List[Any]:


     '''simple docstring'''




     kwargs.update(self.special_tokens_map						)
     return self.tokenizer_class.from_pretrained(self.tmpdirname				,						**lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    str				,						**lowerCAmelCase       :    int						)   ->    Optional[int]:


     '''simple docstring'''




     kwargs.update(self.special_tokens_map						)
     return RobertaTokenizerFast.from_pretrained(self.tmpdirname				,						**lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    Tuple				,						lowerCAmelCase       :    Union[str, Any]						)   ->    Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[int]   	="lower newer"
     SCREAMING_SNAKE_CASE_:       Any   	="lower newer"
     return input_text, output_text



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	=self.tokenizer_class(self.vocab_file				,						self.merges_file				,						**self.special_tokens_map						)
     SCREAMING_SNAKE_CASE_:       List[Any]   	="lower newer"
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
     SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer.tokenize(lowerCAmelCase_						)  # , add_prefix_space=True)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	=tokens + [tokenizer.unk_token]
     SCREAMING_SNAKE_CASE_:       Tuple   	=[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)				,						lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    Dict						)   ->    Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Dict   	=self.get_tokenizer()

     self.assertListEqual(tokenizer.encode("""Hello world!"""				,						add_special_tokens=lowerCAmelCase_						)				,						[0, 3_1414, 232, 328, 2]						)
     self.assertListEqual(
         tokenizer.encode("""Hello world! cécé herlolip 418"""				,						add_special_tokens=lowerCAmelCase_						)				,						[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]				,						)



  @slow
  def 		lowerCamelCase__				(  self       :    Any						)   ->    Dict:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.tokenizer_class.from_pretrained("""roberta-base"""						)

     SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer.encode("""sequence builders"""				,						add_special_tokens=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       str   	=tokenizer.encode("""multi-sequence build"""				,						add_special_tokens=lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer.encode(
         """sequence builders"""				,						add_special_tokens=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer.encode(
         """sequence builders"""				,						"""multi-sequence build"""				,						add_special_tokens=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       List[Any]   	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_				,						lowerCAmelCase_						)

     assert encoded_sentence == encoded_text_from_decode
     assert encoded_pair == encoded_pair_from_decode



  def 		lowerCamelCase__				(  self       :    List[Any]						)   ->    List[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       str   	=self.get_tokenizer()

     SCREAMING_SNAKE_CASE_:       Optional[int]   	="Encode this sequence."
     SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer.byte_encoder[" ".encode("""utf-8"""						)[0]]

     # Testing encoder arguments
     SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
     self.assertNotEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Any   	=tokenizer.convert_ids_to_tokens(encoded[0]						)[0]
     self.assertEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     tokenizer.add_special_tokens({"""bos_token""": """<s>"""}						)
     SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       int   	=tokenizer.convert_ids_to_tokens(encoded[1]						)[0]
     self.assertNotEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     # Testing spaces after special tokens
     SCREAMING_SNAKE_CASE_:       List[str]   	="<mask>"
     tokenizer.add_special_tokens(
         {"""mask_token""": AddedToken(lowerCAmelCase_				,						lstrip=lowerCAmelCase_				,						rstrip=lowerCAmelCase_						)}						)  # mask token has a left space
     SCREAMING_SNAKE_CASE_:       Any   	=tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	="Encode <mask> sequence"
     SCREAMING_SNAKE_CASE_:       List[Any]   	="Encode <mask>sequence"

     SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer.encode(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       int   	=encoded.index(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       str   	=tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
     self.assertEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer.encode(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=encoded.index(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]						)[0]
     self.assertNotEqual(lowerCAmelCase_				,						lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    Optional[int]						)   ->    List[Any]:


     '''simple docstring'''




     pass



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    int:


     '''simple docstring'''




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
           SCREAMING_SNAKE_CASE_:       Tuple   	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       int   	=self.tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	="A, <mask> AllenNLP sentence."
           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer_r.encode_plus(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						return_token_type_ids=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Optional[Any]   	=tokenizer_p.encode_plus(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						return_token_type_ids=lowerCAmelCase_						)

           # token_type_ids should put 0 everywhere
           self.assertEqual(sum(tokens_r["""token_type_ids"""]						)				,						sum(tokens_p["""token_type_ids"""]						)						)

           # attention_mask should put 1 everywhere, so sum over length should be 1
           self.assertEqual(
               sum(tokens_r["""attention_mask"""]						) / len(tokens_r["""attention_mask"""]						)				,						sum(tokens_p["""attention_mask"""]						) / len(tokens_p["""attention_mask"""]						)				,						)

           SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]						)
           SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]						)

           # Rust correctly handles the space before the mask while python doesnt
           self.assertSequenceEqual(tokens_p["""input_ids"""]				,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]						)
           self.assertSequenceEqual(tokens_r["""input_ids"""]				,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]						)

           self.assertSequenceEqual(
               lowerCAmelCase_				,						["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]						)
           self.assertSequenceEqual(
               lowerCAmelCase_				,						["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]						)



  def 		lowerCamelCase__				(  self       :    str						)   ->    Tuple:


     '''simple docstring'''




     for trim_offsets, add_prefix_space in itertools.product([True, False]				,						repeat=2						):
        SCREAMING_SNAKE_CASE_:       List[str]   	=self.rust_tokenizer_class.from_pretrained(
            self.tmpdirname				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)

        SCREAMING_SNAKE_CASE_:       str   	=json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()						)
        SCREAMING_SNAKE_CASE_:       Optional[Any]   	=json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()						)

        self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]				,						lowerCAmelCase_						)

        self.assertEqual(post_processor_state["""add_prefix_space"""]				,						lowerCAmelCase_						)
        self.assertEqual(post_processor_state["""trim_offsets"""]				,						lowerCAmelCase_						)





  def 		lowerCamelCase__				(  self       :    List[str]						)   ->    Union[str, Any]:


     '''simple docstring'''




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
           SCREAMING_SNAKE_CASE_:       List[str]   	="hello"  # `hello` is a token in the vocabulary of `pretrained_name`
           SCREAMING_SNAKE_CASE_:       Optional[int]   	=f'''{text_of_1_token} {text_of_1_token}'''

           SCREAMING_SNAKE_CASE_:       Tuple   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Optional[Any]   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(len(lowerCAmelCase_						) + 1, len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       List[str]   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[Any]   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(len(lowerCAmelCase_						) + 1, len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(len(lowerCAmelCase_						), len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       Dict   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(len(lowerCAmelCase_						), len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       Optional[int]   	=f''' {text}'''

           # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
           #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
           # )
           # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
           # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
           # self.assertEqual(
           #     encoding.offset_mapping[1],
           #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
           # )

           SCREAMING_SNAKE_CASE_:       str   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       str   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(1, 1 + len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(1 + len(lowerCAmelCase_						) + 1, 1 + len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       Tuple   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Any   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, 1 + len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(1 + len(lowerCAmelCase_						), 1 + len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)

           SCREAMING_SNAKE_CASE_:       str   	=self.rust_tokenizer_class.from_pretrained(
               lowerCAmelCase_				,						use_fast=lowerCAmelCase_				,						add_prefix_space=lowerCAmelCase_				,						trim_offsets=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[Any]   	=tokenizer_r(lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           self.assertEqual(encoding.offset_mapping[0]				,						(0, 1 + len(lowerCAmelCase_						))						)
           self.assertEqual(
               encoding.offset_mapping[1]				,						(1 + len(lowerCAmelCase_						), 1 + len(lowerCAmelCase_						) + 1 + len(lowerCAmelCase_						))				,						)











"""simple docstring"""
# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


lowerCamelCase_					=      get_logger(__name__)







class 			UpperCamelCase_       :
	__magic_name__	      =				'''dummy_data'''
	__magic_name__	      =				'''datasets'''
	__magic_name__	      =				False





	def __init__(      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Union[Version, str]	,	lowerCAmelCase_					: Optional[str] = None	,	lowerCAmelCase_					: bool = False	,	lowerCAmelCase_					: bool = True	,	lowerCAmelCase_					: Optional[List[Callable]] = None	,	)						->			Tuple:
				UpperCAmelCase_     : Optional[int]			     =					0
				UpperCAmelCase_     : int			     =					dataset_name
				UpperCAmelCase_     : Optional[int]			     =					cache_dir
				UpperCAmelCase_     : Tuple			     =					use_local_dummy_data
				UpperCAmelCase_     : int			     =					config
				# download_callbacks take a single url as input
				UpperCAmelCase_     : List[Callable]			     =					download_callbacks or []
				# if False, it doesn't load existing files and it returns the paths of the dummy files relative
				# to the dummy_data zip file root
				UpperCAmelCase_     : Optional[Any]			     =					load_existing_dummy_data

				# TODO(PVP, QL) might need to make this more general
				UpperCAmelCase_     : Dict			     =					str(lowerCAmelCase_						)
				# to be downloaded
				UpperCAmelCase_     : Optional[Any]			     =					None
				UpperCAmelCase_     : int			     =					None





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			str:
				if self._dummy_file is None:
							UpperCAmelCase_     : List[str]			     =					self.download_dummy_data()
				return self._dummy_file





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			int:
				if self.config is not None:
							# structure is dummy / config_name / version_name
							return os.path.join("dummy"	,	self.config.name	,	self.version_name						)
				# structure is dummy / version_name
				return os.path.join("dummy"	,	self.version_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				return os.path.join(self.dummy_data_folder	,	"dummy_data.zip"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Tuple:
				UpperCAmelCase_     : int			     =					(
				    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
				)

				UpperCAmelCase_     : Union[str, Any]			     =					cached_path(
				    lowerCAmelCase_	,	cache_dir=self.cache_dir	,	extract_compressed_file=lowerCAmelCase_	,	force_extract=lowerCAmelCase_						)

				return os.path.join(lowerCAmelCase_	,	self.dummy_file_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			int:
				return os.path.join(self.datasets_scripts_dir	,	self.dataset_name	,	self.dummy_zip_file						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[Any]:
				if self._bucket_url is None:
							UpperCAmelCase_     : Union[str, Any]			     =					hf_github_url(self.dataset_name	,	self.dummy_zip_file.replace(os.sep	,	"/"						)						)
				return self._bucket_url





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Optional[Any]:
				# return full path if its a dir
				if os.path.isdir(self.dummy_file						):
							return self.dummy_file
				# else cut off path to file -> example `xsum`.
				return "/".join(self.dummy_file.replace(os.sep	,	"/"						).split("/"						)[:-1]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: List[str]	,	*lowerCAmelCase_					: List[Any]						)						->			Optional[int]:
				if self.load_existing_dummy_data:
							# dummy data is downloaded and tested
							UpperCAmelCase_     : Dict			     =					self.dummy_file
				else:
							# dummy data cannot be downloaded and only the path to dummy file is returned
							UpperCAmelCase_     : Optional[int]			     =					self.dummy_file_name

				# special case when data_url is a dict
				if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							return self.create_dummy_data_dict(lowerCAmelCase_	,	lowerCAmelCase_						)
				elif isinstance(lowerCAmelCase_	,	(list, tuple)						):
							return self.create_dummy_data_list(lowerCAmelCase_	,	lowerCAmelCase_						)
				else:
							return self.create_dummy_data_single(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]	,	lowerCAmelCase_					: int	,	*lowerCAmelCase_					: Union[str, Any]						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: Tuple						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: Union[str, Any]	,	*lowerCAmelCase_					: Tuple	,	**lowerCAmelCase_					: Tuple						)						->			Union[str, Any]:
				return path





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			List[str]:
				return {}





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: List[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Dict			     =					{}
				for key, single_urls in data_url.items():
							for download_callback in self.download_callbacks:
										if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
													for single_url in single_urls:
																download_callback(lowerCAmelCase_						)
										else:
													UpperCAmelCase_     : Tuple			     =					single_urls
													download_callback(lowerCAmelCase_						)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
							if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
										UpperCAmelCase_     : List[str]			     =					[os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						) for x in single_urls]
							else:
										UpperCAmelCase_     : Optional[int]			     =					single_urls
										UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						)
							UpperCAmelCase_     : int			     =					value

				# make sure that values are unique
				if all(isinstance(lowerCAmelCase_	,	lowerCAmelCase_						) for i in dummy_data_dict.values()						) and len(set(dummy_data_dict.values()						)						) < len(
				    dummy_data_dict.values()						):
							# append key to value to make its name unique
							UpperCAmelCase_     : List[str]			     =					{key: value + key for key, value in dummy_data_dict.items()}

				return dummy_data_dict





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: List[Any]	,	lowerCAmelCase_					: Optional[int]						)						->			Dict:
				UpperCAmelCase_     : str			     =					[]
				# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
				UpperCAmelCase_     : int			     =					all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}"	,	lowerCAmelCase_						)						) for url in data_url						)
				UpperCAmelCase_     : Union[str, Any]			     =					all(
				    url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"						) for url in data_url						)
				if data_url and (is_tf_records or is_pubmed_records):
							UpperCAmelCase_     : Tuple			     =					[data_url[0]] * len(lowerCAmelCase_						)
				for single_url in data_url:
							for download_callback in self.download_callbacks:
										download_callback(lowerCAmelCase_						)
							# we force the name of each key to be the last file / folder name of the url path
							# if the url has arguments, we need to encode them with urllib.parse.quote_plus
							UpperCAmelCase_     : Dict			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(single_url.split("/"						)[-1]						)						)
							dummy_data_list.append(lowerCAmelCase_						)
				return dummy_data_list





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: Optional[Any]	,	lowerCAmelCase_					: str						)						->			Optional[int]:
				for download_callback in self.download_callbacks:
							download_callback(lowerCAmelCase_						)
				# we force the name of each key to be the last file / folder name of the url path
				# if the url has arguments, we need to encode them with urllib.parse.quote_plus
				UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(data_url.split("/"						)[-1]						)						)
				if os.path.exists(lowerCAmelCase_						) or not self.load_existing_dummy_data:
							return value
				else:
							# Backward compatibility, maybe deprecate at one point.
							# For many datasets with single url calls to dl_manager.download_and_extract,
							# the dummy_data.zip file is actually the zipped downloaded file
							# while now we expected the dummy_data.zip file to be a directory containing
							# the downloaded file.
							return path_to_dummy_data





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			int:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: Dict						)						->			Optional[Any]:
				def _iter_archive_members(lowerCAmelCase_					: Dict						):
							# this preserves the order of the members inside the ZIP archive
							UpperCAmelCase_     : str			     =					Path(self.dummy_file						).parent
							UpperCAmelCase_     : Optional[Any]			     =					path.relative_to(lowerCAmelCase_						)
							with ZipFile(self.local_path_to_dummy_data						) as zip_file:
										UpperCAmelCase_     : str			     =					zip_file.namelist()
							for member in members:
										if member.startswith(relative_path.as_posix()						):
													yield dummy_parent_path.joinpath(lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					Path(lowerCAmelCase_						)
				UpperCAmelCase_     : List[Any]			     =					_iter_archive_members(lowerCAmelCase_						) if self.use_local_dummy_data else path.rglob("*"						)
				for file_path in file_paths:
							if file_path.is_file() and not file_path.name.startswith((".", "__")						):
										yield file_path.relative_to(lowerCAmelCase_						).as_posix(), file_path.open("rb"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: Tuple						)						->			str:
				if not isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							UpperCAmelCase_     : str			     =					[paths]
				for path in paths:
							if os.path.isfile(lowerCAmelCase_						):
										if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
													return
										yield path
							else:
										for dirpath, dirnames, filenames in os.walk(lowerCAmelCase_						):
													if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
																continue
													dirnames.sort()
													for filename in sorted(lowerCAmelCase_						):
																if filename.startswith((".", "__")						):
																			continue
																yield os.path.join(lowerCAmelCase_	,	lowerCAmelCase_						)



