

"""simple docstring"""





import collections
import importlib.util
import os
import re
from pathlib import Path


__UpperCamelCase  :      List[Any]							      =		'''src/transformers'''


# Matches is_xxx_available()
__UpperCamelCase  :      Union[str, Any]							      =		re.compile(R'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
__UpperCamelCase  :      Optional[Any]							      =		re.compile(R'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__UpperCamelCase  :      int							      =		re.compile(R'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
__UpperCamelCase  :      Optional[int]							      =		re.compile(R'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
__UpperCamelCase  :      str							      =		re.compile(R'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__UpperCamelCase  :      Optional[int]							      =		re.compile(R'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
__UpperCamelCase  :      Tuple							      =		re.compile('''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
__UpperCamelCase  :      Optional[int]							      =		re.compile('''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
__UpperCamelCase  :      str							      =		re.compile(R'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
__UpperCamelCase  :      Tuple							      =		re.compile(R'''^\s*try:''')
# Catches a line with else:
__UpperCamelCase  :      Tuple							      =		re.compile(R'''^\s*else:''')

def   __SCREAMING_SNAKE_CASE  (      A_ ):
	if _re_test_backend.search(A_ ) is None:
		return None
	lowerCAmelCase__	:						Optional[Any]  				=	[b[0] for b in _re_backend.findall(A_ )]
	backends.sort()
	return "_and_".join(A_ )

def   __SCREAMING_SNAKE_CASE  (      A_ ):
	with open(A_						,     '''r'''						,     encoding='''utf-8'''						,     newline='''\n''' ) as f:
		lowerCAmelCase__	:						Union[str, Any]  				=	f.readlines()

	lowerCAmelCase__	:						Optional[int]  				=	0
	while line_index < len(A_ ) and not lines[line_index].startswith('''_import_structure = {''' ):
		line_index += 1

	# If this is a traditional init, just return.
	if line_index >= len(A_ ):
		return None

	# First grab the objects without a specific backend in _import_structure
	lowerCAmelCase__	:						Optional[int]  				=	[]
	while not lines[line_index].startswith('''if TYPE_CHECKING''' ) and find_backend(lines[line_index] ) is None:
		lowerCAmelCase__	:						Any  				=	lines[line_index]
		# If we have everything on a single line, let's deal with it.
		if _re_one_line_import_struct.search(A_ ):
			lowerCAmelCase__	:						Tuple  				=	_re_one_line_import_struct.search(A_ ).groups()[0]
			lowerCAmelCase__	:						int  				=	re.findall('''\[([^\]]+)\]'''						,     A_ )
			for imp in imports:
				objects.extend([obj[1:-1] for obj in imp.split(''', ''' )] )
			line_index += 1
			continue
		lowerCAmelCase__	:						Dict  				=	_re_import_struct_key_value.search(A_ )
		if single_line_import_search is not None:
			lowerCAmelCase__	:						Union[str, Any]  				=	[obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', ''' ) if len(A_ ) > 0]
			objects.extend(A_ )
		elif line.startswith(''' ''' * 8 + '''"''' ):
			objects.append(line[9:-3] )
		line_index += 1

	lowerCAmelCase__	:						Dict  				=	{'''none''': objects}
	# Let's continue with backend-specific objects in _import_structure
	while not lines[line_index].startswith('''if TYPE_CHECKING''' ):
		# If the line is an if not is_backend_available, we grab all objects associated.
		lowerCAmelCase__	:						Optional[Any]  				=	find_backend(lines[line_index] )
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1] ) is None:
			lowerCAmelCase__	:						List[str]  				=	None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index] ) is None:
				line_index += 1

			line_index += 1

			lowerCAmelCase__	:						Optional[Any]  				=	[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index] ) <= 1 or lines[line_index].startswith(''' ''' * 4 ):
				lowerCAmelCase__	:						List[str]  				=	lines[line_index]
				if _re_import_struct_add_one.search(A_ ) is not None:
					objects.append(_re_import_struct_add_one.search(A_ ).groups()[0] )
				elif _re_import_struct_add_many.search(A_ ) is not None:
					lowerCAmelCase__	:						Dict  				=	_re_import_struct_add_many.search(A_ ).groups()[0].split(''', ''' )
					lowerCAmelCase__	:						List[str]  				=	[obj[1:-1] for obj in imports if len(A_ ) > 0]
					objects.extend(A_ )
				elif _re_between_brackets.search(A_ ) is not None:
					lowerCAmelCase__	:						Optional[Any]  				=	_re_between_brackets.search(A_ ).groups()[0].split(''', ''' )
					lowerCAmelCase__	:						str  				=	[obj[1:-1] for obj in imports if len(A_ ) > 0]
					objects.extend(A_ )
				elif _re_quote_object.search(A_ ) is not None:
					objects.append(_re_quote_object.search(A_ ).groups()[0] )
				elif line.startswith(''' ''' * 8 + '''"''' ):
					objects.append(line[9:-3] )
				elif line.startswith(''' ''' * 12 + '''"''' ):
					objects.append(line[13:-3] )
				line_index += 1

			lowerCAmelCase__	:						int  				=	objects
		else:
			line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
	lowerCAmelCase__	:						Optional[Any]  				=	[]
	while (
	    line_index < len(A_ )
	    and find_backend(lines[line_index] ) is None
	    and not lines[line_index].startswith('''else''' )
	):
		lowerCAmelCase__	:						str  				=	lines[line_index]
		lowerCAmelCase__	:						List[Any]  				=	_re_import.search(A_ )
		if single_line_import_search is not None:
			objects.extend(single_line_import_search.groups()[0].split(''', ''' ) )
		elif line.startswith(''' ''' * 8 ):
			objects.append(line[8:-2] )
		line_index += 1

	lowerCAmelCase__	:						Tuple  				=	{'''none''': objects}
	# Let's continue with backend-specific objects
	while line_index < len(A_ ):
		# If the line is an if is_backend_available, we grab all objects associated.
		lowerCAmelCase__	:						List[Any]  				=	find_backend(lines[line_index] )
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1] ) is None:
			lowerCAmelCase__	:						Any  				=	None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index] ) is None:
				line_index += 1

			line_index += 1

			lowerCAmelCase__	:						Optional[Any]  				=	[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index] ) <= 1 or lines[line_index].startswith(''' ''' * 8 ):
				lowerCAmelCase__	:						str  				=	lines[line_index]
				lowerCAmelCase__	:						str  				=	_re_import.search(A_ )
				if single_line_import_search is not None:
					objects.extend(single_line_import_search.groups()[0].split(''', ''' ) )
				elif line.startswith(''' ''' * 12 ):
					objects.append(line[12:-2] )
				line_index += 1

			lowerCAmelCase__	:						List[Any]  				=	objects
		else:
			line_index += 1

	return import_dict_objects, type_hint_objects

def   __SCREAMING_SNAKE_CASE  (      A_						,     A_ ):

	def find_duplicates(A_ ):
		return [k for k, v in collections.Counter(A_ ).items() if v > 1]

	if list(import_dict_objects.keys() ) != list(type_hint_objects.keys() ):
		return ["Both sides of the init do not have the same backends!"]

	lowerCAmelCase__	:						str  				=	[]
	for key in import_dict_objects.keys():
		lowerCAmelCase__	:						Any  				=	find_duplicates(import_dict_objects[key] )
		if duplicate_imports:
			errors.append(f'Duplicate _import_structure definitions for: {duplicate_imports}' )
		lowerCAmelCase__	:						List[Any]  				=	find_duplicates(type_hint_objects[key] )
		if duplicate_type_hints:
			errors.append(f'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}' )

		if sorted(set(import_dict_objects[key] ) ) != sorted(set(type_hint_objects[key] ) ):
			lowerCAmelCase__	:						Tuple  				=	'''base imports''' if key == '''none''' else f'{key} backend'
			errors.append(f'Differences for {name}:' )
			for a in type_hint_objects[key]:
				if a not in import_dict_objects[key]:
					errors.append(f'  {a} in TYPE_HINT but not in _import_structure.' )
			for a in import_dict_objects[key]:
				if a not in type_hint_objects[key]:
					errors.append(f'  {a} in _import_structure but not in TYPE_HINT.' )
	return errors

def   __SCREAMING_SNAKE_CASE  (      ):
	lowerCAmelCase__	:						Optional[int]  				=	[]
	for root, _, files in os.walk(A_ ):
		if "__init__.py" in files:
			lowerCAmelCase__	:						str  				=	os.path.join(A_						,     '''__init__.py''' )
			lowerCAmelCase__	:						Optional[int]  				=	parse_init(A_ )
			if objects is not None:
				lowerCAmelCase__	:						Optional[int]  				=	analyze_results(*A_ )
				if len(A_ ) > 0:
					lowerCAmelCase__	:						Dict  				=	f'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
					failures.append('''\n'''.join(A_ ) )
	if len(A_ ) > 0:
		raise ValueError('''\n\n'''.join(A_ ) )

def   __SCREAMING_SNAKE_CASE  (      ):
	lowerCAmelCase__	:						Optional[Any]  				=	[]
	for path, directories, files in os.walk(A_ ):
		for folder in directories:
			# Ignore private modules
			if folder.startswith('''_''' ):
				directories.remove(A_ )
				continue
			# Ignore leftovers from branches (empty folders apart from pycache)
			if len(list((Path(A_ ) / folder).glob('''*.py''' ) ) ) == 0:
				continue
			lowerCAmelCase__	:						Any  				=	str((Path(A_ ) / folder).relative_to(A_ ) )
			lowerCAmelCase__	:						Optional[int]  				=	short_path.replace(os.path.sep						,     '''.''' )
			submodules.append(A_ )
		for fname in files:
			if fname == "__init__.py":
				continue
			lowerCAmelCase__	:						Any  				=	str((Path(A_ ) / fname).relative_to(A_ ) )
			lowerCAmelCase__	:						str  				=	short_path.replace('''.py'''						,     '''''' ).replace(os.path.sep						,     '''.''' )
			if len(submodule.split('''.''' ) ) == 1:
				submodules.append(A_ )
	return submodules


__UpperCamelCase  :      str							      =		[
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]

def   __SCREAMING_SNAKE_CASE  (      ):
	# This is to make sure the transformers module imported is the one in the repo.
	lowerCAmelCase__	:						Union[str, Any]  				=	importlib.util.spec_from_file_location(
	    '''transformers'''						,     os.path.join(A_						,     '''__init__.py''' )						,     submodule_search_locations=[PATH_TO_TRANSFORMERS]						,     )
	lowerCAmelCase__	:						Any  				=	spec.loader.load_module()

	lowerCAmelCase__	:						List[str]  				=	[
	    module
	    for module in get_transformers_submodules()
	    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
	]
	if len(A_ ) > 0:
		lowerCAmelCase__	:						List[str]  				=	'''\n'''.join(f'- {module}' for module in module_not_registered )
		raise ValueError(
		    '''The following submodules are not properly registered in the main init of Transformers:\n'''
		    f'{list_of_modules}\n'
		    '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.''' )


if __name__ == "__main__":
							check_all_inits()
							check_submodules()







"""simple docstring"""





import numpy as np
import torch
from torch.utils.data import Dataset, IterableDataset

from ..utils.generic import ModelOutput







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				List[Any]   ,lowercase_			:				Tuple   ,lowercase_			:				Dict   ,lowercase_			:				str		):
						lowerCAmelCase__	:						int  				=	dataset
						lowerCAmelCase__	:						List[str]  				=	process
						lowerCAmelCase__	:						Dict  				=	params







					def __len__(       self			:				Any		):
						return len(self.dataset		)







					def __getitem__(       self			:				Union[str, Any]   ,lowercase_			:				List[Any]		):
						lowerCAmelCase__	:						Union[str, Any]  				=	self.dataset[i]
						lowerCAmelCase__	:						Optional[Any]  				=	self.process(lowercase_   ,**self.params		)
						return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				Optional[int]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				List[Any]   ,lowercase_			:				Optional[Any]   ,lowercase_			:				Tuple=None		):
						lowerCAmelCase__	:						List[Any]  				=	loader
						lowerCAmelCase__	:						int  				=	infer
						lowerCAmelCase__	:						List[str]  				=	params
						if loader_batch_size == 1:
							# Let's spare some time by deactivating altogether
							lowerCAmelCase__	:						int  				=	None
						lowerCAmelCase__	:						Dict  				=	loader_batch_size

						# Internal bookkeeping
						lowerCAmelCase__	:						Optional[Any]  				=	None
						lowerCAmelCase__	:						Optional[int]  				=	None







					def __len__(       self			:				Union[str, Any]		):
						return len(self.loader		)







					def __iter__(       self			:				List[Any]		):
						lowerCAmelCase__	:						List[Any]  				=	iter(self.loader		)
						return self







					def        __lowerCAmelCase (       self			:				Tuple		):
						if isinstance(self._loader_batch_data   ,torch.Tensor		):
							# Batch data is simple tensor, just fetch the slice
							lowerCAmelCase__	:						Tuple  				=	self._loader_batch_data[self._loader_batch_index]
						else:
							# Batch data is assumed to be BaseModelOutput (or dict)
							lowerCAmelCase__	:						int  				=	{}
							for k, element in self._loader_batch_data.items():
								if isinstance(lowercase_   ,lowercase_		):
									# Convert ModelOutput to tuple first
									lowerCAmelCase__	:						List[Any]  				=	element.to_tuple()
									if isinstance(element[0]   ,torch.Tensor		):
										lowerCAmelCase__	:						List[Any]  				=	tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
									elif isinstance(element[0]   ,np.ndarray		):
										lowerCAmelCase__	:						str  				=	tuple(np.expand_dims(el[self._loader_batch_index]   ,0		) for el in element		)
									continue
								if k in {"hidden_states", "past_key_values", "attentions"} and isinstance(lowercase_   ,lowercase_		):
									# Those are stored as lists of tensors so need specific unbatching.
									if isinstance(element[0]   ,torch.Tensor		):
										lowerCAmelCase__	:						Dict  				=	tuple(el[self._loader_batch_index].unsqueeze(0		) for el in element		)
									elif isinstance(element[0]   ,np.ndarray		):
										lowerCAmelCase__	:						Optional[int]  				=	tuple(np.expand_dims(el[self._loader_batch_index]   ,0		) for el in element		)
									continue
								if element is None:
									# This can happen for optional data that get passed around
									lowerCAmelCase__	:						Dict  				=	None
								elif isinstance(element[self._loader_batch_index]   ,torch.Tensor		):
									# Take correct batch data, but make it looked like batch_size=1
									# For compatibility with other methods within transformers

									lowerCAmelCase__	:						str  				=	element[self._loader_batch_index].unsqueeze(0		)
								elif isinstance(element[self._loader_batch_index]   ,np.ndarray		):
									# Take correct batch data, but make it looked like batch_size=1
									# For compatibility with other methods within transformers
									lowerCAmelCase__	:						Tuple  				=	np.expand_dims(element[self._loader_batch_index]   ,0		)
								else:
									# This is typically a list, so no need to `unsqueeze`.
									lowerCAmelCase__	:						int  				=	element[self._loader_batch_index]
            # Recreate the element by reusing the original class to make it look
            # batch_size=1
							lowerCAmelCase__	:						int  				=	self._loader_batch_data.__class__(lowercase_		)
						self._loader_batch_index += 1
						return result







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
							# We are currently unrolling a batch so we just need to return
							# the current item within a batch
							return self.loader_batch_item()

						# We're out of items within a batch
						lowerCAmelCase__	:						Dict  				=	next(self.iterator		)
						lowerCAmelCase__	:						List[Any]  				=	self.infer(lowercase_   ,**self.params		)
						# We now have a batch of "inferred things".
						if self.loader_batch_size is not None:
							# Try to infer the size of the batch
							if isinstance(lowercase_   ,torch.Tensor		):
								lowerCAmelCase__	:						int  				=	processed
							else:
								lowerCAmelCase__	:						Union[str, Any]  				=	list(processed.keys()		)[0]
								lowerCAmelCase__	:						Union[str, Any]  				=	processed[key]
							if isinstance(lowercase_   ,lowercase_		):
								lowerCAmelCase__	:						List[Any]  				=	len(lowercase_		)
							else:
								lowerCAmelCase__	:						List[str]  				=	first_tensor.shape[0]
							if 0 < observed_batch_size < self.loader_batch_size:
								# could be last batch so we can't unroll as many
								# elements.
								lowerCAmelCase__	:						Optional[Any]  				=	observed_batch_size
							# Setting internal index to unwrap the batch
							lowerCAmelCase__	:						str  				=	processed
							lowerCAmelCase__	:						Any  				=	0
							return self.loader_batch_item()
						else:
							# We're not unrolling batches
							return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				int   ,lowercase_			:				str   ,lowercase_			:				str   ,lowercase_			:				Union[str, Any]   ,lowercase_			:				int=None		):
						super().__init__(lowercase_   ,lowercase_   ,lowercase_		)







					def __iter__(       self			:				List[Any]		):
						lowerCAmelCase__	:						Dict  				=	iter(self.loader		)
						lowerCAmelCase__	:						Tuple  				=	None
						return self







					def        __lowerCAmelCase (       self			:				Optional[int]		):
						if self.subiterator is None:
							lowerCAmelCase__	:						List[Any]  				=	self.infer(next(self.iterator		)   ,**self.params		)
						try:
							# Try to return next item
							lowerCAmelCase__	:						Optional[int]  				=	next(self.subiterator		)
						except StopIteration:
							# When a preprocess iterator ends, we can start lookig at the next item
							# ChunkIterator will keep feeding until ALL elements of iterator
							# all have created their subiterator and have been iterating against.
							#
							# Another way to look at it, is we're basically flattening lists of lists
							# into a single list, but with generators
							lowerCAmelCase__	:						Any  				=	self.infer(next(self.iterator		)   ,**self.params		)
							lowerCAmelCase__	:						int  				=	next(self.subiterator		)
						return processed







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __iter__(       self			:				Tuple		):
						lowerCAmelCase__	:						int  				=	iter(self.loader		)
						return self







					def        __lowerCAmelCase (       self			:				List[Any]		):
						# Extremely similar to PipelineIterator in its unpacking mechanism
						# BUT, we have an extra required item which is the presence of `is_last`
						# That is because everything is flattened by `PipelineChunkIterator` we
						# need to keep track of how to regroup here in the original `process`
						# boundaries so that `process` and `postprocess` see the same data.

						# This iterator accumulates items (possibly while unbatching) until it
						# its a `is_last` and then just passes it on to the caller.
						lowerCAmelCase__	:						Tuple  				=	False
						lowerCAmelCase__	:						str  				=	[]
						if self._loader_batch_index is not None and self._loader_batch_index < self.loader_batch_size:
							while self._loader_batch_index < self.loader_batch_size:
								lowerCAmelCase__	:						Dict  				=	self.loader_batch_item()
								lowerCAmelCase__	:						Optional[Any]  				=	item.pop('''is_last'''		)
								accumulator.append(lowercase_		)
								if is_last:
									return accumulator

						while not is_last:
							lowerCAmelCase__	:						Any  				=	self.infer(next(self.iterator		)   ,**self.params		)
							if self.loader_batch_size is not None:
								if isinstance(lowercase_   ,torch.Tensor		):
									lowerCAmelCase__	:						Tuple  				=	processed
								else:
									lowerCAmelCase__	:						List[Any]  				=	list(processed.keys()		)[0]
									lowerCAmelCase__	:						Union[str, Any]  				=	processed[key]
								if isinstance(lowercase_   ,lowercase_		):
									lowerCAmelCase__	:						Tuple  				=	len(lowercase_		)
								else:
									lowerCAmelCase__	:						str  				=	first_tensor.shape[0]
								if 0 < observed_batch_size < self.loader_batch_size:
									# could be last batch so we can't unroll as many
									# elements.
									lowerCAmelCase__	:						Optional[int]  				=	observed_batch_size
								lowerCAmelCase__	:						Optional[int]  				=	processed
								lowerCAmelCase__	:						Optional[int]  				=	0
								while self._loader_batch_index < self.loader_batch_size:
									lowerCAmelCase__	:						Any  				=	self.loader_batch_item()
									lowerCAmelCase__	:						Optional[Any]  				=	item.pop('''is_last'''		)
									accumulator.append(lowercase_		)
									if is_last:
										return accumulator
							else:
								lowerCAmelCase__	:						Dict  				=	processed
								lowerCAmelCase__	:						Tuple  				=	item.pop('''is_last'''		)
								accumulator.append(lowercase_		)
						return accumulator







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				int   ,lowercase_			:				Dataset   ,lowercase_			:				str		):
						lowerCAmelCase__	:						List[Any]  				=	dataset
						lowerCAmelCase__	:						List[Any]  				=	key







					def __len__(       self			:				List[Any]		):
						return len(self.dataset		)







					def __getitem__(       self			:				str   ,lowercase_			:				Union[str, Any]		):
						return self.dataset[i][self.key]







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				Dict   ,lowercase_			:				Dataset   ,lowercase_			:				str   ,lowercase_			:				str		):
						lowerCAmelCase__	:						str  				=	dataset
						lowerCAmelCase__	:						List[str]  				=	keya
						lowerCAmelCase__	:						Optional[Any]  				=	keya







					def __len__(       self			:				str		):
						return len(self.dataset		)







					def __getitem__(       self			:				Optional[int]   ,lowercase_			:				Union[str, Any]		):
						return {"text": self.dataset[i][self.keya], "text_pair": self.dataset[i][self.keya]}




