



"""simple docstring"""
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class     a		(     __A		,					unittest.TestCase				):
  UpperCamelCase		: Any     =    BertTokenizer
  UpperCamelCase		: int     =    BertTokenizerFast
  UpperCamelCase		: Optional[Any]     =    True
  UpperCamelCase		: List[str]     =    True
  UpperCamelCase		: int     =    filter_non_english



  def 		lowerCamelCase__				(  self       :    Tuple						)   ->    List[Any]:


     '''simple docstring'''




     super().setUp()

     SCREAMING_SNAKE_CASE_:       Tuple   	=[
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     SCREAMING_SNAKE_CASE_:       Any   	=os.path.join(self.tmpdirname				,						VOCAB_FILES_NAMES["""vocab_file"""]						)
     with open(self.vocab_file				,						"""w"""				,						encoding="""utf-8"""						) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)



  def 		lowerCamelCase__				(  self       :    Any				,						lowerCAmelCase       :    str						)   ->    Union[str, Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	="UNwant\u00E9d,running"
     SCREAMING_SNAKE_CASE_:       Any   	="unwanted, running"
     return input_text, output_text



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    List[str]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Any   	=self.tokenizer_class(self.vocab_file						)

     SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer.tokenize("""UNwant\u00E9d,running"""						)
     self.assertListEqual(lowerCAmelCase_				,						["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]						)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_						)				,						[9, 6, 7, 12, 10, 11]						)



  def 		lowerCamelCase__				(  self       :    List[str]						)   ->    Any:


     '''simple docstring'''




     if not self.test_rust_tokenizer:
        return

     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=self.get_tokenizer()
     SCREAMING_SNAKE_CASE_:       List[Any]   	=self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE_:       List[Any]   	="UNwant\u00E9d,running"

     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer.tokenize(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       int   	=rust_tokenizer.tokenize(lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Any   	=tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=rust_tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer.encode(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=rust_tokenizer.encode(lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     # With lower casing
     SCREAMING_SNAKE_CASE_:       Tuple   	=self.get_tokenizer(do_lower_case=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Optional[int]   	=self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       List[Any]   	="UNwant\u00E9d,running"

     SCREAMING_SNAKE_CASE_:       List[Any]   	=tokenizer.tokenize(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Tuple   	=rust_tokenizer.tokenize(lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=rust_tokenizer.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer.encode(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=rust_tokenizer.encode(lowerCAmelCase_						)
     self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    Any						)   ->    str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""						)				,						["""ah""", """\u535A""", """\u63A8""", """zz"""]						)



  def 		lowerCamelCase__				(  self       :    List[str]						)   ->    Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=BasicTokenizer(do_lower_case=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)				,						["""hello""", """!""", """how""", """are""", """you""", """?"""]						)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)				,						["""hello"""]						)



  def 		lowerCamelCase__				(  self       :    Tuple						)   ->    Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       List[str]   	=BasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)				,						["""hällo""", """!""", """how""", """are""", """you""", """?"""]						)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)				,						["""h\u00E9llo"""]						)



  def 		lowerCamelCase__				(  self       :    int						)   ->    int:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Any   	=BasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)				,						["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)				,						["""hello"""]						)



  def 		lowerCamelCase__				(  self       :    List[str]						)   ->    Optional[int]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	=BasicTokenizer(do_lower_case=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)				,						["""hallo""", """!""", """how""", """are""", """you""", """?"""]						)
     self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)				,						["""hello"""]						)



  def 		lowerCamelCase__				(  self       :    Union[str, Any]						)   ->    str:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       List[str]   	=BasicTokenizer(do_lower_case=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """						)				,						["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    List[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Optional[int]   	=BasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)				,						["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)



  def 		lowerCamelCase__				(  self       :    str						)   ->    Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=BasicTokenizer(do_lower_case=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """						)				,						["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]						)



  def 		lowerCamelCase__				(  self       :    Union[str, Any]						)   ->    int:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	=BasicTokenizer(do_lower_case=lowerCAmelCase_				,						never_split=["""[UNK]"""]						)

     self.assertListEqual(
         tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""						)				,						["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]						)



  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    Any:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Tuple   	=BasicTokenizer()
     SCREAMING_SNAKE_CASE_:       Dict   	="a\n'll !!to?'d of, can't."
     SCREAMING_SNAKE_CASE_:       List[str]   	=["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
     self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_						)				,						lowerCAmelCase_						)



  def 		lowerCamelCase__				(  self       :    Dict						)   ->    int:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       int   	=["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

     SCREAMING_SNAKE_CASE_:       Tuple   	={}
     for i, token in enumerate(lowerCAmelCase_						):
        SCREAMING_SNAKE_CASE_:       Optional[int]   	=i
     SCREAMING_SNAKE_CASE_:       Optional[Any]   	=WordpieceTokenizer(vocab=lowerCAmelCase_				,						unk_token="""[UNK]"""						)

     self.assertListEqual(tokenizer.tokenize(""""""						)				,						[]						)

     self.assertListEqual(tokenizer.tokenize("""unwanted running"""						)				,						["""un""", """##want""", """##ed""", """runn""", """##ing"""]						)

     self.assertListEqual(tokenizer.tokenize("""unwantedX running"""						)				,						["""[UNK]""", """runn""", """##ing"""]						)



  def 		lowerCamelCase__				(  self       :    Dict						)   ->    Optional[int]:


     '''simple docstring'''




     self.assertTrue(_is_whitespace(""" """						)						)
     self.assertTrue(_is_whitespace("""\t"""						)						)
     self.assertTrue(_is_whitespace("""\r"""						)						)
     self.assertTrue(_is_whitespace("""\n"""						)						)
     self.assertTrue(_is_whitespace("""\u00A0"""						)						)

     self.assertFalse(_is_whitespace("""A"""						)						)
     self.assertFalse(_is_whitespace("""-"""						)						)



  def 		lowerCamelCase__				(  self       :    Any						)   ->    Optional[int]:


     '''simple docstring'''




     self.assertTrue(_is_control("""\u0005"""						)						)

     self.assertFalse(_is_control("""A"""						)						)
     self.assertFalse(_is_control(""" """						)						)
     self.assertFalse(_is_control("""\t"""						)						)
     self.assertFalse(_is_control("""\r"""						)						)



  def 		lowerCamelCase__				(  self       :    int						)   ->    List[str]:


     '''simple docstring'''




     self.assertTrue(_is_punctuation("""-"""						)						)
     self.assertTrue(_is_punctuation("""$"""						)						)
     self.assertTrue(_is_punctuation("""`"""						)						)
     self.assertTrue(_is_punctuation("""."""						)						)

     self.assertFalse(_is_punctuation("""A"""						)						)
     self.assertFalse(_is_punctuation(""" """						)						)



  def 		lowerCamelCase__				(  self       :    str						)   ->    Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       Dict   	=self.get_tokenizer()
     SCREAMING_SNAKE_CASE_:       List[str]   	=self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_						) for t in ["""Test""", """\xad""", """test"""]]				,						[["""[UNK]"""], [], ["""[UNK]"""]]						)

     self.assertListEqual(
         [rust_tokenizer.tokenize(lowerCAmelCase_						) for t in ["""Test""", """\xad""", """test"""]]				,						[["""[UNK]"""], [], ["""[UNK]"""]]						)



  @slow
  def 		lowerCamelCase__				(  self       :    Optional[Any]						)   ->    Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       List[str]   	=self.tokenizer_class.from_pretrained("""bert-base-uncased"""						)

     SCREAMING_SNAKE_CASE_:       Any   	=tokenizer.encode("""sequence builders"""				,						add_special_tokens=lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer.encode("""multi-sequence build"""				,						add_special_tokens=lowerCAmelCase_						)

     SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_						)
     SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_				,						lowerCAmelCase_						)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]



  def 		lowerCamelCase__				(  self       :    Any						)   ->    str:


     '''simple docstring'''




     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
           SCREAMING_SNAKE_CASE_:       str   	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
           SCREAMING_SNAKE_CASE_:       Tuple   	=tokenizer_r.encode_plus(
               lowerCAmelCase_				,						return_attention_mask=lowerCAmelCase_				,						return_token_type_ids=lowerCAmelCase_				,						return_offsets_mapping=lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_				,						)

           SCREAMING_SNAKE_CASE_:       Optional[int]   	=tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_				,						"""do_lower_case"""						) else False
           SCREAMING_SNAKE_CASE_:       List[Any]   	=(
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), "A"),
                   ((1, 2), ","),
                   ((3, 5), "na"),
                   ((5, 6), "##ï"),
                   ((6, 8), "##ve"),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), "Allen"),
                   ((21, 23), "##NL"),
                   ((23, 24), "##P"),
                   ((25, 33), "sentence"),
                   ((33, 34), "."),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), "a"),
                   ((1, 2), ","),
                   ((3, 8), "naive"),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), "allen"),
                   ((21, 23), "##nl"),
                   ((23, 24), "##p"),
                   ((25, 33), "sentence"),
                   ((33, 34), "."),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]				,						tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]						)						)
           self.assertEqual([e[0] for e in expected_results]				,						tokens["""offset_mapping"""]						)





  def 		lowerCamelCase__				(  self       :    Union[str, Any]						)   ->    Optional[Any]:


     '''simple docstring'''




     SCREAMING_SNAKE_CASE_:       List[Any]   	=["的", "人", "有"]
     SCREAMING_SNAKE_CASE_:       Tuple   	="".join(lowerCAmelCase_						)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
           SCREAMING_SNAKE_CASE_:       Optional[Any]   	=True
           SCREAMING_SNAKE_CASE_:       Any   	=self.tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer_p.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Dict   	=tokenizer_r.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       Any   	=tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

           # it is expected that each Chinese character is not preceded by "##"
           self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)
           self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       List[Any]   	=False
           SCREAMING_SNAKE_CASE_:       Optional[Any]   	=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[Any]   	=self.tokenizer_class.from_pretrained(lowerCAmelCase_				,						**lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       int   	=tokenizer_r.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer_p.encode(lowerCAmelCase_				,						add_special_tokens=lowerCAmelCase_						)

           SCREAMING_SNAKE_CASE_:       Union[str, Any]   	=tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_						)
           SCREAMING_SNAKE_CASE_:       List[str]   	=tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_						)

           # it is expected that only the first Chinese character is not preceded by "##".
           SCREAMING_SNAKE_CASE_:       Tuple   	=[
               f'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_						)
           ]
           self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)
           self.assertListEqual(lowerCAmelCase_				,						lowerCAmelCase_						)











"""simple docstring"""
# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


lowerCamelCase_					=      get_logger(__name__)







class 			UpperCamelCase_       :
	__magic_name__	      =				'''dummy_data'''
	__magic_name__	      =				'''datasets'''
	__magic_name__	      =				False





	def __init__(      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Union[Version, str]	,	lowerCAmelCase_					: Optional[str] = None	,	lowerCAmelCase_					: bool = False	,	lowerCAmelCase_					: bool = True	,	lowerCAmelCase_					: Optional[List[Callable]] = None	,	)						->			Tuple:
				UpperCAmelCase_     : Optional[int]			     =					0
				UpperCAmelCase_     : int			     =					dataset_name
				UpperCAmelCase_     : Optional[int]			     =					cache_dir
				UpperCAmelCase_     : Tuple			     =					use_local_dummy_data
				UpperCAmelCase_     : int			     =					config
				# download_callbacks take a single url as input
				UpperCAmelCase_     : List[Callable]			     =					download_callbacks or []
				# if False, it doesn't load existing files and it returns the paths of the dummy files relative
				# to the dummy_data zip file root
				UpperCAmelCase_     : Optional[Any]			     =					load_existing_dummy_data

				# TODO(PVP, QL) might need to make this more general
				UpperCAmelCase_     : Dict			     =					str(lowerCAmelCase_						)
				# to be downloaded
				UpperCAmelCase_     : Optional[Any]			     =					None
				UpperCAmelCase_     : int			     =					None





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			str:
				if self._dummy_file is None:
							UpperCAmelCase_     : List[str]			     =					self.download_dummy_data()
				return self._dummy_file





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple						)						->			int:
				if self.config is not None:
							# structure is dummy / config_name / version_name
							return os.path.join("dummy"	,	self.config.name	,	self.version_name						)
				# structure is dummy / version_name
				return os.path.join("dummy"	,	self.version_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Optional[int]:
				return os.path.join(self.dummy_data_folder	,	"dummy_data.zip"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			Tuple:
				UpperCAmelCase_     : int			     =					(
				    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
				)

				UpperCAmelCase_     : Union[str, Any]			     =					cached_path(
				    lowerCAmelCase_	,	cache_dir=self.cache_dir	,	extract_compressed_file=lowerCAmelCase_	,	force_extract=lowerCAmelCase_						)

				return os.path.join(lowerCAmelCase_	,	self.dummy_file_name						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			int:
				return os.path.join(self.datasets_scripts_dir	,	self.dataset_name	,	self.dummy_zip_file						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Dict						)						->			Optional[Any]:
				if self._bucket_url is None:
							UpperCAmelCase_     : Union[str, Any]			     =					hf_github_url(self.dataset_name	,	self.dummy_zip_file.replace(os.sep	,	"/"						)						)
				return self._bucket_url





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Optional[Any]:
				# return full path if its a dir
				if os.path.isdir(self.dummy_file						):
							return self.dummy_file
				# else cut off path to file -> example `xsum`.
				return "/".join(self.dummy_file.replace(os.sep	,	"/"						).split("/"						)[:-1]						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: List[str]	,	*lowerCAmelCase_					: List[Any]						)						->			Optional[int]:
				if self.load_existing_dummy_data:
							# dummy data is downloaded and tested
							UpperCAmelCase_     : Dict			     =					self.dummy_file
				else:
							# dummy data cannot be downloaded and only the path to dummy file is returned
							UpperCAmelCase_     : Optional[int]			     =					self.dummy_file_name

				# special case when data_url is a dict
				if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							return self.create_dummy_data_dict(lowerCAmelCase_	,	lowerCAmelCase_						)
				elif isinstance(lowerCAmelCase_	,	(list, tuple)						):
							return self.create_dummy_data_list(lowerCAmelCase_	,	lowerCAmelCase_						)
				else:
							return self.create_dummy_data_single(lowerCAmelCase_	,	lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]	,	lowerCAmelCase_					: int	,	*lowerCAmelCase_					: Union[str, Any]						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Dict	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: Tuple						)						->			Any:
				return self.download_and_extract(lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: Union[str, Any]	,	*lowerCAmelCase_					: Tuple	,	**lowerCAmelCase_					: Tuple						)						->			Union[str, Any]:
				return path





	def      _SCREAMING_SNAKE_CASE      (      self					: List[str]						)						->			List[str]:
				return {}





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: int	,	lowerCAmelCase_					: List[Any]						)						->			List[Any]:
				UpperCAmelCase_     : Dict			     =					{}
				for key, single_urls in data_url.items():
							for download_callback in self.download_callbacks:
										if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
													for single_url in single_urls:
																download_callback(lowerCAmelCase_						)
										else:
													UpperCAmelCase_     : Tuple			     =					single_urls
													download_callback(lowerCAmelCase_						)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
							if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
										UpperCAmelCase_     : List[str]			     =					[os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						) for x in single_urls]
							else:
										UpperCAmelCase_     : Optional[int]			     =					single_urls
										UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(Path(lowerCAmelCase_						).name						)						)
							UpperCAmelCase_     : int			     =					value

				# make sure that values are unique
				if all(isinstance(lowerCAmelCase_	,	lowerCAmelCase_						) for i in dummy_data_dict.values()						) and len(set(dummy_data_dict.values()						)						) < len(
				    dummy_data_dict.values()						):
							# append key to value to make its name unique
							UpperCAmelCase_     : List[str]			     =					{key: value + key for key, value in dummy_data_dict.items()}

				return dummy_data_dict





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: List[Any]	,	lowerCAmelCase_					: Optional[int]						)						->			Dict:
				UpperCAmelCase_     : str			     =					[]
				# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
				UpperCAmelCase_     : int			     =					all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}"	,	lowerCAmelCase_						)						) for url in data_url						)
				UpperCAmelCase_     : Union[str, Any]			     =					all(
				    url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"						) for url in data_url						)
				if data_url and (is_tf_records or is_pubmed_records):
							UpperCAmelCase_     : Tuple			     =					[data_url[0]] * len(lowerCAmelCase_						)
				for single_url in data_url:
							for download_callback in self.download_callbacks:
										download_callback(lowerCAmelCase_						)
							# we force the name of each key to be the last file / folder name of the url path
							# if the url has arguments, we need to encode them with urllib.parse.quote_plus
							UpperCAmelCase_     : Dict			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(single_url.split("/"						)[-1]						)						)
							dummy_data_list.append(lowerCAmelCase_						)
				return dummy_data_list





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: Optional[Any]	,	lowerCAmelCase_					: str						)						->			Optional[int]:
				for download_callback in self.download_callbacks:
							download_callback(lowerCAmelCase_						)
				# we force the name of each key to be the last file / folder name of the url path
				# if the url has arguments, we need to encode them with urllib.parse.quote_plus
				UpperCAmelCase_     : Optional[Any]			     =					os.path.join(lowerCAmelCase_	,	urllib.parse.quote_plus(data_url.split("/"						)[-1]						)						)
				if os.path.exists(lowerCAmelCase_						) or not self.load_existing_dummy_data:
							return value
				else:
							# Backward compatibility, maybe deprecate at one point.
							# For many datasets with single url calls to dl_manager.download_and_extract,
							# the dummy_data.zip file is actually the zipped downloaded file
							# while now we expected the dummy_data.zip file to be a directory containing
							# the downloaded file.
							return path_to_dummy_data





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			int:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Any						)						->			Optional[int]:
				pass





	def      _SCREAMING_SNAKE_CASE      (      self					: Union[str, Any]	,	lowerCAmelCase_					: Dict						)						->			Optional[Any]:
				def _iter_archive_members(lowerCAmelCase_					: Dict						):
							# this preserves the order of the members inside the ZIP archive
							UpperCAmelCase_     : str			     =					Path(self.dummy_file						).parent
							UpperCAmelCase_     : Optional[Any]			     =					path.relative_to(lowerCAmelCase_						)
							with ZipFile(self.local_path_to_dummy_data						) as zip_file:
										UpperCAmelCase_     : str			     =					zip_file.namelist()
							for member in members:
										if member.startswith(relative_path.as_posix()						):
													yield dummy_parent_path.joinpath(lowerCAmelCase_						)

				UpperCAmelCase_     : Optional[Any]			     =					Path(lowerCAmelCase_						)
				UpperCAmelCase_     : List[Any]			     =					_iter_archive_members(lowerCAmelCase_						) if self.use_local_dummy_data else path.rglob("*"						)
				for file_path in file_paths:
							if file_path.is_file() and not file_path.name.startswith((".", "__")						):
										yield file_path.relative_to(lowerCAmelCase_						).as_posix(), file_path.open("rb"						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: Tuple						)						->			str:
				if not isinstance(lowerCAmelCase_	,	lowerCAmelCase_						):
							UpperCAmelCase_     : str			     =					[paths]
				for path in paths:
							if os.path.isfile(lowerCAmelCase_						):
										if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
													return
										yield path
							else:
										for dirpath, dirnames, filenames in os.walk(lowerCAmelCase_						):
													if os.path.basename(lowerCAmelCase_						).startswith((".", "__")						):
																continue
													dirnames.sort()
													for filename in sorted(lowerCAmelCase_						):
																if filename.startswith((".", "__")						):
																			continue
																yield os.path.join(lowerCAmelCase_	,	lowerCAmelCase_						)



