





import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
      import bitsandbytes as bnb
      import torch
      import torch.nn as nn

      from ..pytorch_utils import ConvaD

if is_accelerate_available():
      from accelerate import init_empty_weights
      from accelerate.utils import find_tied_parameters

snake_case								=							logging.get_logger(__name__)

def 				SCREAMING_SNAKE_CASE__							(	snake_case__	:Union[str, Any]      ,				snake_case__	:Optional[int]      ,				snake_case__	:Any      ,				snake_case__	:Union[str, Any]=None      ,				snake_case__	:str=None  )   ->    Optional[Any]:
  # Recurse if needed
  if "." in tensor_name:
    _lowercase						    =			tensor_name.split('.'  )
    for split in splits[:-1]:
      _lowercase						    =			getattr(snake_case__      ,				snake_case__  )
      if new_module is None:
        raise ValueError(F"""{module} has no attribute {split}."""  )
      _lowercase						    =			new_module
    _lowercase						    =			splits[-1]

  if tensor_name not in module._parameters and tensor_name not in module._buffers:
    raise ValueError(F"""{module} does not have a parameter or a buffer named {tensor_name}."""  )
  _lowercase						    =			tensor_name in module._buffers
  _lowercase						    =			getattr(snake_case__      ,				snake_case__  )

  if old_value.device == torch.device('meta'  ) and device not in ["meta", torch.device('meta'  )] and value is None:
    raise ValueError(F"""{tensor_name} is on the meta device, we need a `value` to put in on {device}."""  )

  _lowercase						    =			False
  _lowercase						    =			False
  if is_buffer or not is_bitsandbytes_available():
    _lowercase						    =			False
    _lowercase						    =			False
  else:
    _lowercase						    =			hasattr(bnb.nn      ,				'Params4bit'  ) and isinstance(module._parameters[tensor_name]      ,				bnb.nn.Paramsabit  )
    _lowercase						    =			isinstance(module._parameters[tensor_name]      ,				bnb.nn.IntaParams  )

  if is_abit or is_abit:
    _lowercase						    =			module._parameters[tensor_name]
    if param.device.type != "cuda":
      if value is None:
        _lowercase						    =			old_value.to(snake_case__  )
      elif isinstance(snake_case__      ,				torch.Tensor  ):
        _lowercase						    =			value.to('cpu'  )
        if value.dtype == torch.inta:
          _lowercase						    =			version.parse(importlib.metadata.version('bitsandbytes'  )  ) > version.parse(
              '0.37.2'  )
          if not is_abit_serializable:
            raise ValueError(
                'Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. '
                'Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`.'  )
      else:
        _lowercase						    =			torch.tensor(snake_case__      ,				device='cpu'  )

      # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
      # Since weights are saved in the correct "orientation", we skip transposing when loading.
      if issubclass(module.source_cls      ,				snake_case__  ) and fpaa_statistics is None:
        _lowercase						    =			new_value.T

      _lowercase						    =			old_value.__dict__
      if is_abit:
        _lowercase						    =			bnb.nn.IntaParams(snake_case__      ,				requires_grad=snake_case__      ,				**snake_case__  ).to(snake_case__  )
      elif is_abit:
        _lowercase						    =			bnb.nn.Paramsabit(snake_case__      ,				requires_grad=snake_case__      ,				**snake_case__  ).to(snake_case__  )

      _lowercase						    =			new_value
      if fpaa_statistics is not None:
        setattr(module.weight      ,				'SCB'      ,				fpaa_statistics.to(snake_case__  )  )

  else:
    if value is None:
      _lowercase						    =			old_value.to(snake_case__  )
    elif isinstance(snake_case__      ,				torch.Tensor  ):
      _lowercase						    =			value.to(snake_case__  )
    else:
      _lowercase						    =			torch.tensor(snake_case__      ,				device=snake_case__  )

    if is_buffer:
      _lowercase						    =			new_value
    else:
      _lowercase						    =			nn.Parameter(snake_case__      ,				requires_grad=old_value.requires_grad  )
      _lowercase						    =			new_value

def 				SCREAMING_SNAKE_CASE__							(	snake_case__	:Union[str, Any]      ,				snake_case__	:Any=None      ,				snake_case__	:Union[str, Any]=None      ,				snake_case__	:Optional[Any]=None      ,				snake_case__	:str=False  )   ->    Optional[int]:
  for name, module in model.named_children():
    if current_key_name is None:
      _lowercase						    =			[]
    current_key_name.append(snake_case__  )

    if (isinstance(snake_case__      ,				nn.Linear  ) or isinstance(snake_case__      ,				snake_case__  )) and name not in modules_to_not_convert:
      # Check if the current key is not in the `modules_to_not_convert`
      if not any(key in '.'.join(snake_case__  ) for key in modules_to_not_convert  ):
        with init_empty_weights():
          if isinstance(snake_case__      ,				snake_case__  ):
            _lowercase    ,		_lowercase						    =			module.weight.shape
          else:
            _lowercase						    =			module.in_features
            _lowercase						    =			module.out_features

          if quantization_config.quantization_method() == "llm_int8":
            _lowercase						    =			bnb.nn.LinearabitLt(
                snake_case__      ,				snake_case__      ,				module.bias is not None      ,				has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight      ,				threshold=quantization_config.llm_inta_threshold      ,				)
            _lowercase						    =			True
          else:
            if (
                quantization_config.llm_inta_skip_modules is not None
                and name in quantization_config.llm_inta_skip_modules
            ):
              pass
            else:
              _lowercase						    =			bnb.nn.Linearabit(
                  snake_case__      ,				snake_case__      ,				module.bias is not None      ,				quantization_config.bnb_abit_compute_dtype      ,				compress_statistics=quantization_config.bnb_abit_use_double_quant      ,				quant_type=quantization_config.bnb_abit_quant_type      ,				)
              _lowercase						    =			True
                    # Store the module class in case we need to transpose the weight later
          _lowercase						    =			type(snake_case__  )
          # Force requires grad to False to avoid unexpected errors
          model._modules[name].requires_grad_(snake_case__  )
    if len(list(module.children()  )  ) > 0:
      _lowercase    ,		_lowercase						    =			_replace_with_bnb_linear(
          snake_case__      ,				snake_case__      ,				snake_case__      ,				snake_case__      ,				has_been_replaced=snake_case__      ,				)
    # Remove the last key for recursion
    current_key_name.pop(-1  )
  return model, has_been_replaced

def 				SCREAMING_SNAKE_CASE__							(	snake_case__	:int      ,				snake_case__	:Union[str, Any]=None      ,				snake_case__	:Union[str, Any]=None      ,				snake_case__	:Union[str, Any]=None  )   ->    Optional[Any]:
  _lowercase						    =			['lm_head'] if modules_to_not_convert is None else modules_to_not_convert
  _lowercase    ,		_lowercase						    =			_replace_with_bnb_linear(
      snake_case__      ,				snake_case__      ,				snake_case__      ,				snake_case__  )

  if not has_been_replaced:
    logger.warning(
        'You are loading your model in 8bit or 4bit but no linear modules were found in your model.'
        ' Please double check your model architecture, or submit an issue on github if you think this is'
        ' a bug.'  )

  return model

def 				SCREAMING_SNAKE_CASE__							(	*snake_case__	:Optional[Any]      ,				**snake_case__	:Optional[int]  )   ->    str:
  warnings.warn(
      '`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead'      ,				snake_case__      ,				)
  return replace_with_bnb_linear(*snake_case__      ,				**snake_case__  )

def 				SCREAMING_SNAKE_CASE__							(	*snake_case__	:Dict      ,				**snake_case__	:List[str]  )   ->    Tuple:
  warnings.warn(
      '`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead'      ,				snake_case__      ,				)
  return set_module_quantized_tensor_to_device(*snake_case__      ,				**snake_case__  )







def 				SCREAMING_SNAKE_CASE__							(	snake_case__	:int  )   ->    Any:
  _lowercase						    =			deepcopy(snake_case__  )  # this has 0 cost since it is done inside `init_empty_weights` context manager`
  tied_model.tie_weights()

  _lowercase						    =			find_tied_parameters(snake_case__  )
  # For compatibility with Accelerate < 0.18
  if isinstance(snake_case__      ,				snake_case__  ):
    _lowercase						    =			sum(list(tied_params.values()  )      ,				[]  ) + list(tied_params.keys()  )
  else:
    _lowercase						    =			sum(snake_case__      ,				[]  )
  _lowercase						    =			len(snake_case__  ) > 0

  # Check if it is a base model
  _lowercase						    =			not hasattr(snake_case__      ,				model.base_model_prefix  )

  # Ignore this for base models (BertModel, GPT2Model, etc.)
  if (not has_tied_params) and is_base_model:
    return []

  # otherwise they have an attached head
  _lowercase						    =			list(model.named_children()  )
  _lowercase						    =			[list_modules[-1][0]]

  # add last module together with tied weights
  _lowercase						    =			set(snake_case__  ) - set(snake_case__  )
  _lowercase						    =			list(set(snake_case__  )  ) + list(snake_case__  )

  # remove ".weight" from the keys
  _lowercase						    =			['.weight', '.bias']
  _lowercase						    =			[]
  for name in list_untouched:
    for name_to_remove in names_to_remove:
      if name_to_remove in name:
        _lowercase						    =			name.replace(snake_case__      ,				''  )
    filtered_module_names.append(snake_case__  )

  return filtered_module_names

import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)







class 							lowerCAmelCase__   ( __lowercase						):

						def __init__(      self							,				a=0.01							,				a=10_00					)       ->   Optional[int]:


								'''simple docstring'''

								_UpperCamelCase         =						p_stop
								_UpperCamelCase         =						max_length







						def __iter__(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						0
								_UpperCamelCase         =						False
								while not stop and count < self.max_length:
										yield count
										count += 1
										_UpperCamelCase         =						random.random() < self.p_stop







class 							lowerCAmelCase__   ( unittest.TestCase						):

						def 			A_					(      self							,				a							,				a							,				a=False							,				a=True					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						[
								    BatchSamplerShard(a							,				2							,				a							,				split_batches=a							,				even_batches=a					)
								    for i in range(2					)
								]
								_UpperCamelCase         =						[list(a					) for batch_sampler_shard in batch_sampler_shards]
								if not split_batches:
										self.assertListEqual([len(a					) for shard in batch_sampler_shards]							,				[len(a					) for e in expected]					)
								self.assertListEqual(a							,				a					)

						def 			A_					(      self					)       ->   Any:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is a round multiple of batch size but not total batch size.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1, 0]], [[1, 0, 1]]]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a					)

						def 			A_					(      self					)       ->   Tuple:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size or num_processes.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], [[0, 1]]]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

						def 			A_					(      self					)       ->   str:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is a round multiple of batch size but not total batch size.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], []]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

						def 			A_					(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size or num_processes.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

						def 			A_					(      self					)       ->   List[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
								_UpperCamelCase         =						[BatchSamplerShard(a							,				2							,				a							,				even_batches=a					) for i in range(2					)]

								self.assertEqual(len(batch_sampler_shards[0]					)							,				3					)
								self.assertEqual(len(batch_sampler_shards[1]					)							,				2					)

								self.assertListEqual(list(batch_sampler_shards[0]					)							,				[[0, 1, 2], [5, 6, 7, 8], [12, 13]]					)
								self.assertListEqual(list(batch_sampler_shards[1]					)							,				[[3, 4], [9, 10, 11]]					)

						def 			A_					(      self							,				a							,				a							,				a							,				a=False							,				a=2							,				a=False					)       ->   Any:


								'''simple docstring'''

								random.seed(a					)
								_UpperCamelCase         =						list(a					)

								_UpperCamelCase         =						[
								    IterableDatasetShard(
								        a							,				batch_size=a							,				drop_last=a							,				num_processes=a							,				process_index=a							,				split_batches=a							,				)
								    for i in range(a					)
								]
								_UpperCamelCase         =						[]
								for iterable_dataset_shard in iterable_dataset_shards:
										# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
										random.seed(a					)
										iterable_dataset_lists.append(list(a					)					)

								_UpperCamelCase         =						batch_size // num_processes if split_batches else batch_size
								# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
								_UpperCamelCase         =						iterable_dataset_lists[0]
								for l in iterable_dataset_lists[1:]:
										self.assertEqual(len(a					)							,				len(a					)					)
										self.assertTrue(len(a					) % shard_batch_size == 0					)

								_UpperCamelCase         =						[]
								for idx in range(0							,				len(a					)							,				a					):
										for l in iterable_dataset_lists:
												observed += l[idx : idx + shard_batch_size]

								if not drop_last:
										while len(a					) < len(a					):
												reference += reference
								self.assertListEqual(a							,				reference[: len(a					)]					)

						def 			A_					(      self					)       ->   int:


								'''simple docstring'''

								_UpperCamelCase         =						42
								_UpperCamelCase         =						RandomIterableDataset()

								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)

								# Edge case with a very small dataset
								_UpperCamelCase         =						RandomIterableDataset(max_length=2					)

								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)

						def 			A_					(      self					)       ->   Any:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(16					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						SkipBatchSampler(a							,				2					)
								self.assertListEqual(list(a					)							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   Dict:


								'''simple docstring'''

								_UpperCamelCase         =						SkipDataLoader(list(range(16					)					)							,				batch_size=4							,				skip_batches=2					)
								self.assertListEqual([t.tolist() for t in dataloader]							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   Tuple:


								'''simple docstring'''

								_UpperCamelCase         =						DataLoader(list(range(16					)					)							,				batch_size=4					)
								_UpperCamelCase         =						skip_first_batches(a							,				num_batches=2					)
								self.assertListEqual([t.tolist() for t in new_dataloader]							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   List[str]:


								'''simple docstring'''

								_UpperCamelCase         =						DataLoaderShard(list(range(16					)					)							,				batch_size=4					)
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)

								# Test it also works on the second iteration
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)







						def 			A_					(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								Accelerator()
								_UpperCamelCase         =						DataLoaderDispatcher(range(16					)							,				batch_size=4					)
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)

								# Test it also works on the second iteration
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)



