




import collections
import importlib.util
import os
import re
from pathlib import Path


_UpperCAmelCase		:  str	=   "src/transformers"


# Matches is_xxx_available()
_UpperCAmelCase		:  Union[str, Any]	=   re.compile(r"is\_([a-z_]*)_available()")
# Catches a one-line _import_struct = {xxx}
_UpperCAmelCase		:  Tuple	=   re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
_UpperCAmelCase		:  int	=   re.compile(r"\s+\"\S*\":\s+\[([^\]]*)\]")
# Catches a line if not is_foo_available
_UpperCAmelCase		:  Optional[int]	=   re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")
# Catches a line _import_struct["bla"].append("foo")
_UpperCAmelCase		:  List[str]	=   re.compile(r"^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
_UpperCAmelCase		:  List[Any]	=   re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")
# Catches a line with an object between quotes and a comma:     "MyModel",
_UpperCAmelCase		:  int	=   re.compile("^\s+\"([^\"]+)\",")
# Catches a line with objects between brackets only:    ["foo", "bar"],
_UpperCAmelCase		:  Union[str, Any]	=   re.compile("^\s+\[([^\]]+)\]")
# Catches a line with from foo import bar, bla, boo
_UpperCAmelCase		:  Dict	=   re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
# Catches a line with try:
_UpperCAmelCase		:  List[Any]	=   re.compile(r"^\s*try:")
# Catches a line with else:
_UpperCAmelCase		:  Optional[int]	=   re.compile(r"^\s*else:")



def 					UpperCAmelCase__     (   lowerCamelCase      ):
  if _re_test_backend.search(__lowerCamelCase      ) is None:
    return None
  lowercase			:Union[str, Any]							=  [b[0] for b in _re_backend.findall(__lowerCamelCase      )]
  backends.sort()
  return "_and_".join(__lowerCamelCase      )



def 					UpperCAmelCase__     (   lowerCamelCase      ):
  with open(__lowerCamelCase,	"r",	encoding="utf-8",	newline="\n"      ) as f:
    lowercase			:str							=  f.readlines()

  lowercase			:List[str]							=  0
  while line_index < len(__lowerCamelCase      ) and not lines[line_index].startswith("_import_structure = {"      ):
    line_index += 1

  # If this is a traditional init, just return.
  if line_index >= len(__lowerCamelCase      ):
    return None

  # First grab the objects without a specific backend in _import_structure
  lowercase			:List[str]							=  []
  while not lines[line_index].startswith("if TYPE_CHECKING"      ) and find_backend(lines[line_index]      ) is None:
    lowercase			:List[str]							=  lines[line_index]
    # If we have everything on a single line, let's deal with it.
    if _re_one_line_import_struct.search(__lowerCamelCase      ):
      lowercase			:List[Any]							=  _re_one_line_import_struct.search(__lowerCamelCase      ).groups()[0]
      lowercase			:Tuple							=  re.findall("\[([^\]]+)\]",	__lowerCamelCase      )
      for imp in imports:
        objects.extend([obj[1:-1] for obj in imp.split(", "      )]      )
      line_index += 1
      continue
    lowercase			:Dict							=  _re_import_struct_key_value.search(__lowerCamelCase      )
    if single_line_import_search is not None:
      lowercase			:int							=  [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", "      ) if len(__lowerCamelCase      ) > 0]
      objects.extend(__lowerCamelCase      )
    elif line.startswith(" " * 8 + "\""      ):
      objects.append(line[9:-3]      )
    line_index += 1

  lowercase			:str							=  {"none": objects}
  # Let's continue with backend-specific objects in _import_structure
  while not lines[line_index].startswith("if TYPE_CHECKING"      ):
    # If the line is an if not is_backend_available, we grab all objects associated.
    lowercase			:Union[str, Any]							=  find_backend(lines[line_index]      )
    # Check if the backend declaration is inside a try block:
    if _re_try.search(lines[line_index - 1]      ) is None:
      lowercase			:Dict							=  None

    if backend is not None:
      line_index += 1

      # Scroll until we hit the else block of try-except-else
      while _re_else.search(lines[line_index]      ) is None:
        line_index += 1

      line_index += 1

      lowercase			:Optional[int]							=  []
      # Until we unindent, add backend objects to the list
      while len(lines[line_index]      ) <= 1 or lines[line_index].startswith(" " * 4      ):
        lowercase			:Union[str, Any]							=  lines[line_index]
        if _re_import_struct_add_one.search(__lowerCamelCase      ) is not None:
          objects.append(_re_import_struct_add_one.search(__lowerCamelCase      ).groups()[0]      )
        elif _re_import_struct_add_many.search(__lowerCamelCase      ) is not None:
          lowercase			:List[str]							=  _re_import_struct_add_many.search(__lowerCamelCase      ).groups()[0].split(", "      )
          lowercase			:Optional[int]							=  [obj[1:-1] for obj in imports if len(__lowerCamelCase      ) > 0]
          objects.extend(__lowerCamelCase      )
        elif _re_between_brackets.search(__lowerCamelCase      ) is not None:
          lowercase			:Optional[int]							=  _re_between_brackets.search(__lowerCamelCase      ).groups()[0].split(", "      )
          lowercase			:Optional[int]							=  [obj[1:-1] for obj in imports if len(__lowerCamelCase      ) > 0]
          objects.extend(__lowerCamelCase      )
        elif _re_quote_object.search(__lowerCamelCase      ) is not None:
          objects.append(_re_quote_object.search(__lowerCamelCase      ).groups()[0]      )
        elif line.startswith(" " * 8 + "\""      ):
          objects.append(line[9:-3]      )
        elif line.startswith(" " * 12 + "\""      ):
          objects.append(line[13:-3]      )
        line_index += 1

      lowercase			:str							=  objects
    else:
      line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
  lowercase			:Optional[Any]							=  []
  while (
      line_index < len(__lowerCamelCase      )
      and find_backend(lines[line_index]      ) is None
      and not lines[line_index].startswith("else"      )
  ):
    lowercase			:Optional[Any]							=  lines[line_index]
    lowercase			:int							=  _re_import.search(__lowerCamelCase      )
    if single_line_import_search is not None:
      objects.extend(single_line_import_search.groups()[0].split(", "      )      )
    elif line.startswith(" " * 8      ):
      objects.append(line[8:-2]      )
    line_index += 1

  lowercase			:List[str]							=  {"none": objects}
  # Let's continue with backend-specific objects
  while line_index < len(__lowerCamelCase      ):
    # If the line is an if is_backend_available, we grab all objects associated.
    lowercase			:Optional[int]							=  find_backend(lines[line_index]      )
    # Check if the backend declaration is inside a try block:
    if _re_try.search(lines[line_index - 1]      ) is None:
      lowercase			:Tuple							=  None

    if backend is not None:
      line_index += 1

      # Scroll until we hit the else block of try-except-else
      while _re_else.search(lines[line_index]      ) is None:
        line_index += 1

      line_index += 1

      lowercase			:Union[str, Any]							=  []
      # Until we unindent, add backend objects to the list
      while len(lines[line_index]      ) <= 1 or lines[line_index].startswith(" " * 8      ):
        lowercase			:Union[str, Any]							=  lines[line_index]
        lowercase			:int							=  _re_import.search(__lowerCamelCase      )
        if single_line_import_search is not None:
          objects.extend(single_line_import_search.groups()[0].split(", "      )      )
        elif line.startswith(" " * 12      ):
          objects.append(line[12:-2]      )
        line_index += 1

      lowercase			:str							=  objects
    else:
      line_index += 1

  return import_dict_objects, type_hint_objects



def 					UpperCAmelCase__     (   lowerCamelCase,	lowerCamelCase      ):

  def find_duplicates(lowerCamelCase      ):
    return [k for k, v in collections.Counter(__lowerCamelCase      ).items() if v > 1]

  if list(import_dict_objects.keys()      ) != list(type_hint_objects.keys()      ):
    return ["Both sides of the init do not have the same backends!"]

  lowercase			:Dict							=  []
  for key in import_dict_objects.keys():
    lowercase			:Optional[Any]							=  find_duplicates(import_dict_objects[key]      )
    if duplicate_imports:
      errors.append(F"Duplicate _import_structure definitions for: {duplicate_imports}"      )
    lowercase			:Tuple							=  find_duplicates(type_hint_objects[key]      )
    if duplicate_type_hints:
      errors.append(F"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"      )

    if sorted(set(import_dict_objects[key]      )      ) != sorted(set(type_hint_objects[key]      )      ):
      lowercase			:Tuple							=  "base imports" if key == "none" else F"{key} backend"
      errors.append(F"Differences for {name}:"      )
      for a in type_hint_objects[key]:
        if a not in import_dict_objects[key]:
          errors.append(F"  {a} in TYPE_HINT but not in _import_structure."      )
      for a in import_dict_objects[key]:
        if a not in type_hint_objects[key]:
          errors.append(F"  {a} in _import_structure but not in TYPE_HINT."      )
  return errors



def 					UpperCAmelCase__     (   ):
  lowercase			:Dict							=  []
  for root, _, files in os.walk(__lowerCamelCase      ):
    if "__init__.py" in files:
      lowercase			:str							=  os.path.join(__lowerCamelCase,	"__init__.py"      )
      lowercase			:Optional[int]							=  parse_init(__lowerCamelCase      )
      if objects is not None:
        lowercase			:str							=  analyze_results(*__lowerCamelCase      )
        if len(__lowerCamelCase      ) > 0:
          lowercase			:Optional[Any]							=  F"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"
          failures.append("\n".join(__lowerCamelCase      )      )
  if len(__lowerCamelCase      ) > 0:
    raise ValueError("\n\n".join(__lowerCamelCase      )      )



def 					UpperCAmelCase__     (   ):
  lowercase			:List[Any]							=  []
  for path, directories, files in os.walk(__lowerCamelCase      ):
    for folder in directories:
      # Ignore private modules
      if folder.startswith("_"      ):
        directories.remove(__lowerCamelCase      )
        continue
      # Ignore leftovers from branches (empty folders apart from pycache)
      if len(list((Path(__lowerCamelCase      ) / folder).glob("*.py"      )      )      ) == 0:
        continue
      lowercase			:Optional[int]							=  str((Path(__lowerCamelCase      ) / folder).relative_to(__lowerCamelCase      )      )
      lowercase			:int							=  short_path.replace(os.path.sep,	"."      )
      submodules.append(__lowerCamelCase      )
    for fname in files:
      if fname == "__init__.py":
        continue
      lowercase			:Union[str, Any]							=  str((Path(__lowerCamelCase      ) / fname).relative_to(__lowerCamelCase      )      )
      lowercase			:List[Any]							=  short_path.replace(".py",	""      ).replace(os.path.sep,	"."      )
      if len(submodule.split("."      )      ) == 1:
        submodules.append(__lowerCamelCase      )
  return submodules


_UpperCAmelCase		:  str	=   [
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
]



def 					UpperCAmelCase__     (   ):
  # This is to make sure the transformers module imported is the one in the repo.
  lowercase			:Optional[int]							=  importlib.util.spec_from_file_location(
      "transformers",	os.path.join(__lowerCamelCase,	"__init__.py"      ),	submodule_search_locations=[PATH_TO_TRANSFORMERS],	)
  lowercase			:List[Any]							=  spec.loader.load_module()

  lowercase			:str							=  [
      module
      for module in get_transformers_submodules()
      if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
  ]
  if len(__lowerCamelCase      ) > 0:
    lowercase			:str							=  "\n".join(F"- {module}" for module in module_not_registered      )
    raise ValueError(
        "The following submodules are not properly registered in the main init of Transformers:\n"
        F"{list_of_modules}\n"
        "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."      )


if __name__ == "__main__":
     check_all_inits()
     check_submodules()





"""simple docstring"""



import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


UpperCAmelCase__          = 'http://www.mocksite.com/file1.txt'
UpperCAmelCase__          = '"text": ["foo", "foo"]'
UpperCAmelCase__          = '6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'

class       lowerCAmelCase__				:
     __a						=					200
     __a						=					{"""Content-Length""": """100"""}
     __a						=					{}







     def      lowercase	(							self			:			List[str]    ,      **_lowerCamelCase			:			List[str]			):
       return [bytes(_lowerCamelCase    ,      '''utf-8'''			)]






def 					_UpperCAmelCase      ( *__lowerCamelCase   :       List[str] ,					**__lowerCamelCase   :       Dict		)     ->					Dict:
  return MockResponse()
@pytest.mark.parametrize('''urls_type''' ,					[str, list, dict]		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Union[str, Any] ,					__lowerCamelCase   :       List[str] ,					__lowerCamelCase   :       str		)     ->					int:
  import requests

  monkeypatch.setattr(__lowerCamelCase ,					'''request''' ,					__lowerCamelCase		)

  _snake_case					     =       URL
  if issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       url
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       [url]
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       {'''train''': url}
  _snake_case					     =       '''dummy'''
  _snake_case					     =       '''downloads'''
  _snake_case					     =       tmp_path
  _snake_case					     =       DownloadConfig(
      cache_dir=os.path.join(__lowerCamelCase ,					__lowerCamelCase		) ,					use_etag=__lowerCamelCase ,					)
  _snake_case					     =       DownloadManager(dataset_name=__lowerCamelCase ,					download_config=__lowerCamelCase		)
  _snake_case					     =       dl_manager.download(__lowerCamelCase		)
  _snake_case					     =       urls
  for downloaded_paths in [downloaded_paths]:
    if isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      _snake_case					     =       [downloaded_paths]
      _snake_case					     =       [urls]
    elif isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      assert "train" in downloaded_paths.keys()
      _snake_case					     =       downloaded_paths.values()
      _snake_case					     =       urls.values()
    assert downloaded_paths
    for downloaded_path, input_url in zip(__lowerCamelCase ,					__lowerCamelCase		):
      assert downloaded_path == dl_manager.downloaded_paths[input_url]
      _snake_case					     =       Path(__lowerCamelCase		)
      _snake_case					     =       downloaded_path.parts
      assert parts[-1] == HASH
      assert parts[-2] == cache_subdir
      assert downloaded_path.exists()
      _snake_case					     =       downloaded_path.read_text()
      assert content == CONTENT
      _snake_case					     =       downloaded_path.with_suffix('''.json'''		)
      assert metadata_downloaded_path.exists()
      _snake_case					     =       json.loads(metadata_downloaded_path.read_text()		)
      assert metadata_content == {"url": URL, "etag": None}
@pytest.mark.parametrize('''paths_type''' ,					[str, list, dict]		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       int ,					__lowerCamelCase   :       str ,					__lowerCamelCase   :       Optional[int]		)     ->					int:
  _snake_case					     =       str(__lowerCamelCase		)
  if issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       filename
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       [filename]
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       {'''train''': filename}
  _snake_case					     =       '''dummy'''
  _snake_case					     =       xz_file.parent
  _snake_case					     =       '''extracted'''
  _snake_case					     =       DownloadConfig(
      cache_dir=__lowerCamelCase ,					use_etag=__lowerCamelCase ,					)
  _snake_case					     =       DownloadManager(dataset_name=__lowerCamelCase ,					download_config=__lowerCamelCase		)
  _snake_case					     =       dl_manager.extract(__lowerCamelCase		)
  _snake_case					     =       paths
  for extracted_paths in [extracted_paths]:
    if isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      _snake_case					     =       [extracted_paths]
      _snake_case					     =       [paths]
    elif isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      assert "train" in extracted_paths.keys()
      _snake_case					     =       extracted_paths.values()
      _snake_case					     =       paths.values()
    assert extracted_paths
    for extracted_path, input_path in zip(__lowerCamelCase ,					__lowerCamelCase		):
      assert extracted_path == dl_manager.extracted_paths[input_path]
      _snake_case					     =       Path(__lowerCamelCase		)
      _snake_case					     =       extracted_path.parts
      assert parts[-1] == hash_url_to_filename(__lowerCamelCase ,					etag=__lowerCamelCase		)
      assert parts[-2] == extracted_subdir
      assert extracted_path.exists()
      _snake_case					     =       extracted_path.read_text()
      _snake_case					     =       text_file.read_text()
      assert extracted_file_content == expected_file_content
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Tuple ,					__lowerCamelCase   :       List[Any]		)     ->					Dict:
  assert path.endswith('''.jsonl'''		)
  for num_items, line in enumerate(__lowerCamelCase ,					start=1		):
    _snake_case					     =       json.loads(line.decode('''utf-8'''		)		)
    assert item.keys() == {"col_1", "col_2", "col_3"}
  assert num_items == 4
@pytest.mark.parametrize('''archive_jsonl''' ,					['''tar_jsonl_path''', '''zip_jsonl_path''']		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Dict ,					__lowerCamelCase   :       str		)     ->					Dict:
  _snake_case					     =       request.getfixturevalue(__lowerCamelCase		)
  _snake_case					     =       DownloadManager()
  for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
    _test_jsonl(__lowerCamelCase ,					__lowerCamelCase		)
  assert num_jsonl == 2
@pytest.mark.parametrize('''archive_nested_jsonl''' ,					['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       str ,					__lowerCamelCase   :       List[Any]		)     ->					Tuple:
  _snake_case					     =       request.getfixturevalue(__lowerCamelCase		)
  _snake_case					     =       DownloadManager()
  for num_tar, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
    for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
      _test_jsonl(__lowerCamelCase ,					__lowerCamelCase		)
  assert num_tar == 1
  assert num_jsonl == 2

def 					_UpperCAmelCase      ( __lowerCamelCase   :       Tuple		)     ->					List[Any]:
  _snake_case					     =       DownloadManager()
  for num_file, file in enumerate(dl_manager.iter_files(__lowerCamelCase		) ,					start=1		):
    assert os.path.basename(__lowerCamelCase		) == ("test.txt" if num_file == 1 else "train.txt")
  assert num_file == 2



