




'''simple docstring'''


import inspect
from typing import Optional, Union

import numpy as np
import PIL
import torch
from torch.nn import functional as F
from torchvision import transforms
from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    DPMSolverMultistepScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
from diffusers.utils import (
    PIL_INTERPOLATION,
    randn_tensor,
)


def     SCREAMING_SNAKE_CASE							(   a_						:				str					,       a_						:				Any					,       a_						:				Optional[Any]      ):






  if isinstance(a_					,       torch.Tensor      ):
    return image
  elif isinstance(a_					,       PIL.Image.Image      ):
    __a								=       [image]

  if isinstance(image[0]					,       PIL.Image.Image      ):
    __a								=       [np.array(i.resize((w, h)					,       resample=PIL_INTERPOLATION['lanczos']      )      )[None, :] for i in image]
    __a								=       np.concatenate(a_					,       axis=0      )
    __a								=       np.array(a_      ).astype(np.floataa      ) / 2_55.0
    __a								=       image.transpose(0					,       3					,       1					,       2      )
    __a								=       2.0 * image - 1.0
    __a								=       torch.from_numpy(a_      )
  elif isinstance(image[0]					,       torch.Tensor      ):
    __a								=       torch.cat(a_					,       dim=0      )
  return image


def     SCREAMING_SNAKE_CASE							(   a_						:				Union[str, Any]					,       a_						:				Any					,       a_						:				Optional[Any]					,       a_						:				int=0.99_95      ):






  if not isinstance(a_					,       np.ndarray      ):
    __a								=       True
    __a								=       va.device
    __a								=       va.cpu().numpy()
    __a								=       va.cpu().numpy()

  __a								=       np.sum(va * va / (np.linalg.norm(a_      ) * np.linalg.norm(a_      ))      )
  if np.abs(a_      ) > DOT_THRESHOLD:
    __a								=       (1 - t) * va + t * va
  else:
    __a								=       np.arccos(a_      )
    __a								=       np.sin(a_      )
    __a								=       theta_a * t
    __a								=       np.sin(a_      )
    __a								=       np.sin(theta_a - theta_t      ) / sin_theta_a
    __a								=       sin_theta_t / sin_theta_a
    __a								=       sa * va + sa * va

  if inputs_are_torch:
    __a								=       torch.from_numpy(a_      ).to(a_      )

  return va


def     SCREAMING_SNAKE_CASE							(   a_						:				Union[str, Any]					,       a_						:				str      ):
  __a								=       F.normalize(a_					,       dim=-1      )
  __a								=       F.normalize(a_					,       dim=-1      )
  return (x - y).norm(dim=-1      ).div(2      ).arcsin().pow(2      ).mul(2      )


def     SCREAMING_SNAKE_CASE							(   a_						:				Optional[int]					,       a_						:				str      ):






  for param in model.parameters():
    __a								=       value





class  __lowercase    (	__lowercase			):







   def __init__(						self   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase=None   ,				UpperCamelCase=None   ,				UpperCamelCase=None   ,				)					->     List[Any]:
     super().__init__()
     self.register_modules(
         vae=UpperCamelCase   ,				text_encoder=UpperCamelCase   ,				clip_model=UpperCamelCase   ,				tokenizer=UpperCamelCase   ,				unet=UpperCamelCase   ,				scheduler=UpperCamelCase   ,				feature_extractor=UpperCamelCase   ,				coca_model=UpperCamelCase   ,				coca_tokenizer=UpperCamelCase   ,				coca_transform=UpperCamelCase   ,				)
     __a								=       (
         feature_extractor.size
         if isinstance(feature_extractor.size   ,				UpperCamelCase   )
         else feature_extractor.size['shortest_edge']
     )
     __a								=       transforms.Normalize(mean=feature_extractor.image_mean   ,				std=feature_extractor.image_std   )
     set_requires_grad(self.text_encoder   ,				UpperCamelCase   )
     set_requires_grad(self.clip_model   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase = "auto"   )					->     Dict:

     if slice_size == "auto":
       # half the attention head size is usually a good trade-off between
       # speed and memory
       __a								=       self.unet.config.attention_head_dim // 2
     self.unet.set_attention_slice(UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     str:
     self.enable_attention_slicing(UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     List[str]:
     set_requires_grad(self.vae   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Optional[Any]:
     set_requires_grad(self.vae   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Any:
     set_requires_grad(self.unet   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     set_requires_grad(self.unet   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )					->     Optional[Any]:
     __a								=       min(int(num_inference_steps * strength   )   ,				UpperCamelCase   )

     __a								=       max(num_inference_steps - init_timestep   ,				0   )
     __a								=       self.scheduler.timesteps[t_start:]

     return timesteps, num_inference_steps - t_start







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase=None   )					->     Dict:

     if not isinstance(UpperCamelCase   ,				torch.Tensor   ):
       raise ValueError(f"`image` has to be of type `torch.Tensor` but is {type(UpperCamelCase   )}"   )

     __a								=       image.to(device=UpperCamelCase   ,				dtype=UpperCamelCase   )

     if isinstance(UpperCamelCase   ,				UpperCamelCase   ):
       __a								=       [
           self.vae.encode(image[i : i + 1]   ).latent_dist.sample(generator[i]   ) for i in range(UpperCamelCase   )
       ]
       __a								=       torch.cat(UpperCamelCase   ,				dim=0   )
     else:
       __a								=       self.vae.encode(UpperCamelCase   ).latent_dist.sample(UpperCamelCase   )

     # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
     __a								=       0.18_215 * init_latents
     __a								=       init_latents.repeat_interleave(UpperCamelCase   ,				dim=0   )

     __a								=       randn_tensor(init_latents.shape   ,				generator=UpperCamelCase   ,				device=UpperCamelCase   ,				dtype=UpperCamelCase   )

     # get latents
     __a								=       self.scheduler.add_noise(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )
     __a								=       init_latents

     return latents







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   )					->     Optional[Any]:
     __a								=       self.coca_transform(UpperCamelCase   ).unsqueeze(0   )
     with torch.no_grad(), torch.cuda.amp.autocast():
       __a								=       self.coca_model.generate(transformed_image.to(device=self.device   ,				dtype=self.coca_model.dtype   )   )
     __a								=       self.coca_tokenizer.decode(generated[0].cpu().numpy()   )
     return generated.split('<end_of_text>'   )[0].replace('<start_of_text>'   ,				''   ).rstrip(' .,'   )







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase   )					->     int:
     __a								=       self.feature_extractor.preprocess(UpperCamelCase   )
     __a								=       torch.from_numpy(clip_image_input['pixel_values'][0]   ).unsqueeze(0   ).to(self.device   ).half()
     __a								=       self.clip_model.get_image_features(UpperCamelCase   )
     __a								=       image_embeddings_clip / image_embeddings_clip.norm(p=2   ,				dim=-1   ,				keepdim=UpperCamelCase   )
     __a								=       image_embeddings_clip.repeat_interleave(UpperCamelCase   ,				dim=0   )
     return image_embeddings_clip







   @torch.enable_grad()
   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				)					->     int:
     __a								=       latents.detach().requires_grad_()

     __a								=       self.scheduler.scale_model_input(UpperCamelCase   ,				UpperCamelCase   )

     # predict the noise residual
     __a								=       self.unet(UpperCamelCase   ,				UpperCamelCase   ,				encoder_hidden_states=UpperCamelCase   ).sample

     if isinstance(self.scheduler   ,				(PNDMScheduler, DDIMScheduler, DPMSolverMultistepScheduler)   ):
       __a								=       self.scheduler.alphas_cumprod[timestep]
       __a								=       1 - alpha_prod_t
       # compute predicted original sample from predicted noise also called
       # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
       __a								=       (latents - beta_prod_t ** 0.5 * noise_pred) / alpha_prod_t ** 0.5

       __a								=       torch.sqrt(UpperCamelCase   )
       __a								=       pred_original_sample * (fac) + latents * (1 - fac)
     elif isinstance(self.scheduler   ,				UpperCamelCase   ):
       __a								=       self.scheduler.sigmas[index]
       __a								=       latents - sigma * noise_pred
     else:
       raise ValueError(f"scheduler type {type(self.scheduler   )} not supported"   )

     # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
     __a								=       1 / 0.18_215 * sample
     __a								=       self.vae.decode(UpperCamelCase   ).sample
     __a								=       (image / 2 + 0.5).clamp(0   ,				1   )

     __a								=       transforms.Resize(self.feature_extractor_size   )(UpperCamelCase   )
     __a								=       self.normalize(UpperCamelCase   ).to(latents.dtype   )

     __a								=       self.clip_model.get_image_features(UpperCamelCase   )
     __a								=       image_embeddings_clip / image_embeddings_clip.norm(p=2   ,				dim=-1   ,				keepdim=UpperCamelCase   )

     __a								=       spherical_dist_loss(UpperCamelCase   ,				UpperCamelCase   ).mean() * clip_guidance_scale

     __a								=       -torch.autograd.grad(UpperCamelCase   ,				UpperCamelCase   )[0]

     if isinstance(self.scheduler   ,				UpperCamelCase   ):
       __a								=       latents.detach() + grads * (sigma**2)
       __a								=       noise_pred_original
     else:
       __a								=       noise_pred_original - torch.sqrt(UpperCamelCase   ) * grads
     return noise_pred, latents







   @torch.no_grad()
   def __call__(						self   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase = None   ,				UpperCamelCase = None   ,				UpperCamelCase = 512   ,				UpperCamelCase = 512   ,				UpperCamelCase = 0.6   ,				UpperCamelCase = 50   ,				UpperCamelCase = 7.5   ,				UpperCamelCase = 1   ,				UpperCamelCase = 0.0   ,				UpperCamelCase = 100   ,				UpperCamelCase = None   ,				UpperCamelCase = "pil"   ,				UpperCamelCase = True   ,				UpperCamelCase = 0.8   ,				UpperCamelCase = 0.1   ,				UpperCamelCase = 0.1   ,				)					->     Any:

     if isinstance(UpperCamelCase   ,				UpperCamelCase   ) and len(UpperCamelCase   ) != batch_size:
       raise ValueError(f"You have passed {batch_size} batch_size, but only {len(UpperCamelCase   )} generators."   )

     if height % 8 != 0 or width % 8 != 0:
       raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}."   )

     if isinstance(UpperCamelCase   ,				torch.Generator   ) and batch_size > 1:
       __a								=       [generator] + [None] * (batch_size - 1)

     __a								=       [
         ('model', self.coca_model is None),
         ('tokenizer', self.coca_tokenizer is None),
         ('transform', self.coca_transform is None),
     ]
     __a								=       [x[0] for x in coca_is_none if x[1]]
     __a								=       ', '.join(UpperCamelCase   )
     # generate prompts with coca model if prompt is None
     if content_prompt is None:
       if len(UpperCamelCase   ):
         raise ValueError(
             f"Content prompt is None and CoCa [{coca_is_none_str}] is None."
             f"Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."   )
       __a								=       self.get_image_description(UpperCamelCase   )
     if style_prompt is None:
       if len(UpperCamelCase   ):
         raise ValueError(
             f"Style prompt is None and CoCa [{coca_is_none_str}] is None."
             f" Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."   )
       __a								=       self.get_image_description(UpperCamelCase   )

     # get prompt text embeddings for content and style
     __a								=       self.tokenizer(
         UpperCamelCase   ,				padding='max_length'   ,				max_length=self.tokenizer.model_max_length   ,				truncation=UpperCamelCase   ,				return_tensors='pt'   ,				)
     __a								=       self.text_encoder(content_text_input.input_ids.to(self.device   )   )[0]

     __a								=       self.tokenizer(
         UpperCamelCase   ,				padding='max_length'   ,				max_length=self.tokenizer.model_max_length   ,				truncation=UpperCamelCase   ,				return_tensors='pt'   ,				)
     __a								=       self.text_encoder(style_text_input.input_ids.to(self.device   )   )[0]

     __a								=       slerp(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )

     # duplicate text embeddings for each generation per prompt
     __a								=       text_embeddings.repeat_interleave(UpperCamelCase   ,				dim=0   )

     # set timesteps
     __a								=       'offset' in set(inspect.signature(self.scheduler.set_timesteps   ).parameters.keys()   )
     __a								=       {}
     if accepts_offset:
       __a								=       1

     self.scheduler.set_timesteps(UpperCamelCase   ,				**UpperCamelCase   )
     # Some schedulers like PNDM have timesteps as arrays
     # It's more optimized to move all timesteps to correct device beforehand
     self.scheduler.timesteps.to(self.device   )

     __a						,							__a								=       self.get_timesteps(UpperCamelCase   ,				UpperCamelCase   ,				self.device   )
     __a								=       timesteps[:1].repeat(UpperCamelCase   )

     # Preprocess image
     __a								=       preprocess(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )
     __a								=       self.prepare_latents(
         UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				text_embeddings.dtype   ,				self.device   ,				UpperCamelCase   )

     __a								=       preprocess(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )
     __a								=       self.prepare_latents(
         UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				text_embeddings.dtype   ,				self.device   ,				UpperCamelCase   )

     __a								=       slerp(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )

     if clip_guidance_scale > 0:
       __a								=       self.get_clip_image_embeddings(UpperCamelCase   ,				UpperCamelCase   )
       __a								=       self.get_clip_image_embeddings(UpperCamelCase   ,				UpperCamelCase   )
       __a								=       slerp(
           UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   )

     # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
     # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
     # corresponds to doing no classifier free guidance.
     __a								=       guidance_scale > 1.0
     # get unconditional embeddings for classifier free guidance
     if do_classifier_free_guidance:
       __a								=       content_text_input.input_ids.shape[-1]
       __a								=       self.tokenizer(['']   ,				padding='max_length'   ,				max_length=UpperCamelCase   ,				return_tensors='pt'   )
       __a								=       self.text_encoder(uncond_input.input_ids.to(self.device   )   )[0]
       # duplicate unconditional embeddings for each generation per prompt
       __a								=       uncond_embeddings.repeat_interleave(UpperCamelCase   ,				dim=0   )

       # For classifier free guidance, we need to do two forward passes.
       # Here we concatenate the unconditional and text embeddings into a single batch
       # to avoid doing two forward passes
       __a								=       torch.cat([uncond_embeddings, text_embeddings]   )

     # get the initial random noise unless the user supplied it

     # Unlike in other pipelines, latents need to be generated in the target device
     # for 1-to-1 results reproducibility with the CompVis implementation.
     # However this currently doesn't work in `mps`.
     __a								=       (batch_size, self.unet.config.in_channels, height // 8, width // 8)
     __a								=       text_embeddings.dtype
     if latents is None:
       if self.device.type == "mps":
         # randn does not work reproducibly on mps
         __a								=       torch.randn(UpperCamelCase   ,				generator=UpperCamelCase   ,				device='cpu'   ,				dtype=UpperCamelCase   ).to(
             self.device   )
       else:
         __a								=       torch.randn(UpperCamelCase   ,				generator=UpperCamelCase   ,				device=self.device   ,				dtype=UpperCamelCase   )
     else:
       if latents.shape != latents_shape:
         raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}"   )
       __a								=       latents.to(self.device   )

     # scale the initial noise by the standard deviation required by the scheduler
     __a								=       latents * self.scheduler.init_noise_sigma

     # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
     # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
     # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
     # and should be between [0, 1]
     __a								=       'eta' in set(inspect.signature(self.scheduler.step   ).parameters.keys()   )
     __a								=       {}
     if accepts_eta:
       __a								=       eta

     # check if the scheduler accepts generator
     __a								=       'generator' in set(inspect.signature(self.scheduler.step   ).parameters.keys()   )
     if accepts_generator:
       __a								=       generator

     with self.progress_bar(total=UpperCamelCase   ):
       for i, t in enumerate(UpperCamelCase   ):
         # expand the latents if we are doing classifier free guidance
         __a								=       torch.cat([latents] * 2   ) if do_classifier_free_guidance else latents
         __a								=       self.scheduler.scale_model_input(UpperCamelCase   ,				UpperCamelCase   )

         # predict the noise residual
         __a								=       self.unet(UpperCamelCase   ,				UpperCamelCase   ,				encoder_hidden_states=UpperCamelCase   ).sample

         # perform classifier free guidance
         if do_classifier_free_guidance:
           __a						,							__a								=       noise_pred.chunk(2   )
           __a								=       noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

         # perform clip guidance
         if clip_guidance_scale > 0:
           __a								=       (
               text_embeddings.chunk(2   )[1] if do_classifier_free_guidance else text_embeddings
           )
           __a						,							__a								=       self.cond_fn(
               UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				)

         # compute the previous noisy sample x_t -> x_t-1
         __a								=       self.scheduler.step(UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				**UpperCamelCase   ).prev_sample

        # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
     __a								=       1 / 0.18_215 * latents
     __a								=       self.vae.decode(UpperCamelCase   ).sample

     __a								=       (image / 2 + 0.5).clamp(0   ,				1   )
     __a								=       image.cpu().permute(0   ,				2   ,				3   ,				1   ).numpy()

     if output_type == "pil":
       __a								=       self.numpy_to_pil(UpperCamelCase   )

     if not return_dict:
       return (image, None)

     return StableDiffusionPipelineOutput(images=UpperCamelCase   ,				nsfw_content_detected=UpperCamelCase   )





import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)







class 							lowerCAmelCase__   ( __lowercase						):

						def __init__(      self							,				a=0.01							,				a=10_00					)       ->   Optional[int]:


								'''simple docstring'''

								_UpperCamelCase         =						p_stop
								_UpperCamelCase         =						max_length







						def __iter__(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						0
								_UpperCamelCase         =						False
								while not stop and count < self.max_length:
										yield count
										count += 1
										_UpperCamelCase         =						random.random() < self.p_stop







class 							lowerCAmelCase__   ( unittest.TestCase						):

						def 			A_					(      self							,				a							,				a							,				a=False							,				a=True					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						[
								    BatchSamplerShard(a							,				2							,				a							,				split_batches=a							,				even_batches=a					)
								    for i in range(2					)
								]
								_UpperCamelCase         =						[list(a					) for batch_sampler_shard in batch_sampler_shards]
								if not split_batches:
										self.assertListEqual([len(a					) for shard in batch_sampler_shards]							,				[len(a					) for e in expected]					)
								self.assertListEqual(a							,				a					)

						def 			A_					(      self					)       ->   Any:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is a round multiple of batch size but not total batch size.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1, 0]], [[1, 0, 1]]]
								self.check_batch_sampler_shards(a							,				a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a					)

						def 			A_					(      self					)       ->   Tuple:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size or num_processes.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], [[0, 1]]]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a					)

						def 			A_					(      self					)       ->   str:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=3							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is a round multiple of batch size but not total batch size.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
								# num_processes batch.
								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(20					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
								    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
								]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], []]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=3							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				even_batches=a					)

						def 			A_					(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(24					)							,				batch_size=4							,				drop_last=a					)
								# Expected shouldn't change
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size.
								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(22					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is not a round multiple of batch size or num_processes.
								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(21					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[
								    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
								    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
								]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								# Check the shards when the dataset is very small.
								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[[0, 1]], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

								_UpperCamelCase         =						BatchSampler(range(2					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						[[], []]
								self.check_batch_sampler_shards(a							,				a							,				split_batches=a							,				even_batches=a					)

						def 			A_					(      self					)       ->   List[Any]:


								'''simple docstring'''

								_UpperCamelCase         =						[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
								_UpperCamelCase         =						[BatchSamplerShard(a							,				2							,				a							,				even_batches=a					) for i in range(2					)]

								self.assertEqual(len(batch_sampler_shards[0]					)							,				3					)
								self.assertEqual(len(batch_sampler_shards[1]					)							,				2					)

								self.assertListEqual(list(batch_sampler_shards[0]					)							,				[[0, 1, 2], [5, 6, 7, 8], [12, 13]]					)
								self.assertListEqual(list(batch_sampler_shards[1]					)							,				[[3, 4], [9, 10, 11]]					)

						def 			A_					(      self							,				a							,				a							,				a							,				a=False							,				a=2							,				a=False					)       ->   Any:


								'''simple docstring'''

								random.seed(a					)
								_UpperCamelCase         =						list(a					)

								_UpperCamelCase         =						[
								    IterableDatasetShard(
								        a							,				batch_size=a							,				drop_last=a							,				num_processes=a							,				process_index=a							,				split_batches=a							,				)
								    for i in range(a					)
								]
								_UpperCamelCase         =						[]
								for iterable_dataset_shard in iterable_dataset_shards:
										# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
										random.seed(a					)
										iterable_dataset_lists.append(list(a					)					)

								_UpperCamelCase         =						batch_size // num_processes if split_batches else batch_size
								# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
								_UpperCamelCase         =						iterable_dataset_lists[0]
								for l in iterable_dataset_lists[1:]:
										self.assertEqual(len(a					)							,				len(a					)					)
										self.assertTrue(len(a					) % shard_batch_size == 0					)

								_UpperCamelCase         =						[]
								for idx in range(0							,				len(a					)							,				a					):
										for l in iterable_dataset_lists:
												observed += l[idx : idx + shard_batch_size]

								if not drop_last:
										while len(a					) < len(a					):
												reference += reference
								self.assertListEqual(a							,				reference[: len(a					)]					)

						def 			A_					(      self					)       ->   int:


								'''simple docstring'''

								_UpperCamelCase         =						42
								_UpperCamelCase         =						RandomIterableDataset()

								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)

								# Edge case with a very small dataset
								_UpperCamelCase         =						RandomIterableDataset(max_length=2					)

								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)
								self.check_iterable_dataset_shards(a							,				a							,				batch_size=4							,				drop_last=a							,				split_batches=a					)

						def 			A_					(      self					)       ->   Any:


								'''simple docstring'''

								_UpperCamelCase         =						BatchSampler(range(16					)							,				batch_size=4							,				drop_last=a					)
								_UpperCamelCase         =						SkipBatchSampler(a							,				2					)
								self.assertListEqual(list(a					)							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   Dict:


								'''simple docstring'''

								_UpperCamelCase         =						SkipDataLoader(list(range(16					)					)							,				batch_size=4							,				skip_batches=2					)
								self.assertListEqual([t.tolist() for t in dataloader]							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   Tuple:


								'''simple docstring'''

								_UpperCamelCase         =						DataLoader(list(range(16					)					)							,				batch_size=4					)
								_UpperCamelCase         =						skip_first_batches(a							,				num_batches=2					)
								self.assertListEqual([t.tolist() for t in new_dataloader]							,				[[8, 9, 10, 11], [12, 13, 14, 15]]					)

						def 			A_					(      self					)       ->   List[str]:


								'''simple docstring'''

								_UpperCamelCase         =						DataLoaderShard(list(range(16					)					)							,				batch_size=4					)
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)

								# Test it also works on the second iteration
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)







						def 			A_					(      self					)       ->   Optional[Any]:


								'''simple docstring'''

								Accelerator()
								_UpperCamelCase         =						DataLoaderDispatcher(range(16					)							,				batch_size=4					)
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)

								# Test it also works on the second iteration
								for idx, _ in enumerate(a					):
										self.assertEqual(dataloader.end_of_dataloader							,				idx == 3					)



