






import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence([1, 2, 3]    )    )
							self.assertEqual(arr.type       ,    pa.intaa()    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							with self.assertRaises(UpperCAmelCase    ):
													_UpperCAmelCase									=			pa.array(TypedSequence([1, 2, 3]    )       ,    type=pa.intaa()    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							with self.assertRaises(UpperCAmelCase    ):
													_UpperCAmelCase									=			pa.array(TypedSequence([1, 2, 3]       ,    try_type=Value('bool'    )       ,    type=Value('int64'    )    )    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence([1, 2, 3]       ,    type=Value('int32'    )    )    )
							self.assertEqual(arr.type       ,    pa.intaa()    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
													_UpperCAmelCase									=			pa.array(TypedSequence(['foo', 'bar']       ,    type=Value('int64'    )    )    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence([1, 2, 3]       ,    try_type=Value('int32'    )    )    )
							self.assertEqual(arr.type       ,    pa.intaa()    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence(['foo', 'bar']       ,    try_type=Value('int64'    )    )    )
							self.assertEqual(arr.type       ,    pa.string()    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence([[[1, 2, 3]]]       ,    type=ArrayaD((1, 3)       ,    'int64'    )    )    )
							self.assertEqual(arr.type       ,    ArrayaDExtensionType((1, 3)       ,    'int64'    )    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
													_UpperCAmelCase									=			pa.array(TypedSequence(['foo', 'bar']       ,    type=ArrayaD((1, 3)       ,    'int64'    )    )    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence([[[1, 2, 3]]]       ,    try_type=ArrayaD((1, 3)       ,    'int64'    )    )    )
							self.assertEqual(arr.type       ,    ArrayaDExtensionType((1, 3)       ,    'int64'    )    )

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			pa.array(TypedSequence(['foo', 'bar']       ,    try_type=ArrayaD((1, 3)       ,    'int64'    )    )    )
							self.assertEqual(arr.type       ,    pa.string()    )

	@require_pil
	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							import PIL.Image

							_UpperCAmelCase									=			PIL.Image.fromarray(np.arange(10       ,    dtype=np.uinta    ).reshape(2       ,    5    )    )
							with patch(
							    'datasets.arrow_writer.cast_to_python_objects'       ,    side_effect=UpperCAmelCase    ) as mock_cast_to_python_objects:
													_UpperCAmelCase									=			pa.array(TypedSequence([{'path': None, 'bytes': b'image_bytes'}, pil_image]       ,    type=Image()    )    )
													_UpperCAmelCase					, _UpperCAmelCase									=			mock_cast_to_python_objects.call_args_list[-1]
													self.assertIn('optimize_list_casting'       ,    UpperCAmelCase    )
													self.assertFalse(kwargs['optimize_list_casting']    )



def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferReader(__lowerCAmelCase	) if isinstance(__lowerCAmelCase ,		pa.Buffer	) else pa.memory_map(__lowerCAmelCase	)
						_UpperCAmelCase									=			pa.ipc.open_stream(__lowerCAmelCase	)
						_UpperCAmelCase									=			f.read_all()
						assert len(pa_table.to_batches()	) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table


@pytest.mark.parametrize('writer_batch_size' ,		[None, 1, 10]	)
@pytest.mark.parametrize(
    'fields' ,		[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						_UpperCAmelCase									=			pa.schema(__lowerCAmelCase	) if fields else None
						with ArrowWriter(stream=__lowerCAmelCase ,		schema=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase	) as writer:
												writer.write({'col_1': 'foo', 'col_2': 1}	)
												writer.write({'col_1': 'bar', 'col_2': 2}	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_UpperCAmelCase									=			{'col_1': pa.string(), 'col_2': pa.intaa()}
						assert writer._schema == pa.schema(__lowerCAmelCase ,		metadata=writer._schema.metadata	)
						_check_output(output.getvalue() ,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)


def __A    (      )->  Union[str, Any]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						_UpperCAmelCase									=			Features({'labels': ClassLabel(names=['neg', 'pos']	)}	)
						with ArrowWriter(stream=__lowerCAmelCase ,		features=__lowerCAmelCase	) as writer:
												writer.write({'labels': 0}	)
												writer.write({'labels': 1}	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						_UpperCAmelCase									=			pa.BufferReader(output.getvalue()	)
						_UpperCAmelCase									=			pa.ipc.open_stream(__lowerCAmelCase	)
						_UpperCAmelCase									=			f.read_all()
						_UpperCAmelCase									=			pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(__lowerCAmelCase	)


@pytest.mark.parametrize('writer_batch_size' ,		[None, 1, 10]	)
def __A    (      __lowerCAmelCase	)->  Dict:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase ,		hash_salt='split_name' ,		check_duplicates=__lowerCAmelCase ,		) as writer:
												with pytest.raises(__lowerCAmelCase	):
																		writer.write({'col_1': 'foo', 'col_2': 1} ,		key=[1, 2]	)
																		_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()


@pytest.mark.parametrize('writer_batch_size' ,		[None, 2, 10]	)
def __A    (      __lowerCAmelCase	)->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase ,		hash_salt='split_name' ,		check_duplicates=__lowerCAmelCase ,		) as writer:
												with pytest.raises(__lowerCAmelCase	):
																		writer.write({'col_1': 'foo', 'col_2': 1} ,		key=10	)
																		writer.write({'col_1': 'bar', 'col_2': 2} ,		key=10	)
																		_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()


@pytest.mark.parametrize('writer_batch_size' ,		[None, 2, 10]	)
def __A    (      __lowerCAmelCase	)->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase ,		hash_salt='split_name' ,		check_duplicates=__lowerCAmelCase ,		) as writer:
												writer.write({'col_1': 'foo', 'col_2': 1} ,		key=1	)
												writer.write({'col_1': 'bar', 'col_2': 2} ,		key=2	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue() ,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)


@pytest.mark.parametrize('writer_batch_size' ,		[None, 1, 10]	)
@pytest.mark.parametrize(
    'fields' ,		[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Optional[int]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						_UpperCAmelCase									=			pa.schema(__lowerCAmelCase	) if fields else None
						with ArrowWriter(stream=__lowerCAmelCase ,		schema=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase	) as writer:
												writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}	)
												writer.write_batch({'col_1': [], 'col_2': []}	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_UpperCAmelCase									=			{'col_1': pa.string(), 'col_2': pa.intaa()}
						assert writer._schema == pa.schema(__lowerCAmelCase ,		metadata=writer._schema.metadata	)
						_check_output(output.getvalue() ,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)


@pytest.mark.parametrize('writer_batch_size' ,		[None, 1, 10]	)
@pytest.mark.parametrize(
    'fields' ,		[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Optional[Any]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						_UpperCAmelCase									=			pa.schema(__lowerCAmelCase	) if fields else None
						with ArrowWriter(stream=__lowerCAmelCase ,		schema=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase	) as writer:
												writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}	)	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_UpperCAmelCase									=			{'col_1': pa.string(), 'col_2': pa.intaa()}
						assert writer._schema == pa.schema(__lowerCAmelCase ,		metadata=writer._schema.metadata	)
						_check_output(output.getvalue() ,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)


@pytest.mark.parametrize('writer_batch_size' ,		[None, 1, 10]	)
@pytest.mark.parametrize(
    'fields' ,		[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  List[str]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						_UpperCAmelCase									=			pa.schema(__lowerCAmelCase	) if fields else None
						with ArrowWriter(stream=__lowerCAmelCase ,		schema=__lowerCAmelCase ,		writer_batch_size=__lowerCAmelCase	) as writer:
												writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}	)	)
												writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}	)	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_UpperCAmelCase									=			{'col_1': pa.string(), 'col_2': pa.intaa()}
						assert writer._schema == pa.schema(__lowerCAmelCase ,		metadata=writer._schema.metadata	)
						_check_output(output.getvalue() ,		expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)


def __A    (      )->  int:





						"""simple docstring"""



						with tempfile.TemporaryDirectory() as tmp_dir:
												_UpperCAmelCase									=			{'col_1': pa.string(), 'col_2': pa.intaa()}
												_UpperCAmelCase									=			os.path.join(__lowerCAmelCase ,		'test.arrow'	)
												with ArrowWriter(path=__lowerCAmelCase ,		schema=pa.schema(__lowerCAmelCase	)	) as writer:
																		writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}	)
																		_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(__lowerCAmelCase ,		metadata=writer._schema.metadata	)
												_check_output(__lowerCAmelCase ,		1	)


def __A    (      __lowerCAmelCase	)->  Tuple:





						"""simple docstring"""



						if pa.types.is_list(__lowerCAmelCase	):
												return get_base_dtype(arr_type.value_type	)
						else:
												return arr_type


def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Dict:





						"""simple docstring"""



						if isinstance(lst[0] ,		__lowerCAmelCase	):
												change_first_primitive_element_in_list(lst[0] ,		__lowerCAmelCase	)
						else:
												_UpperCAmelCase									=			value


@pytest.mark.parametrize('optimized_int_type, expected_dtype' ,		[(None, pa.intaa()), (Value('int32'	), pa.intaa())]	)
@pytest.mark.parametrize('sequence' ,		[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase ,		__lowerCAmelCase	)->  int:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.array(TypedSequence(__lowerCAmelCase ,		optimized_int_type=__lowerCAmelCase	)	)
						assert get_base_dtype(arr.type	) == expected_dtype


@pytest.mark.parametrize(
    'col, expected_dtype' ,		[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ] ,		)
@pytest.mark.parametrize('sequence' ,		[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase ,		__lowerCAmelCase	)->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.array(OptimizedTypedSequence(__lowerCAmelCase ,		col=__lowerCAmelCase	)	)
						assert get_base_dtype(arr.type	) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												_UpperCAmelCase									=			copy.deepcopy(__lowerCAmelCase	)
												_UpperCAmelCase									=			np.iinfo(expected_dtype.to_pandas_dtype()	).max + 1
												change_first_primitive_element_in_list(__lowerCAmelCase ,		__lowerCAmelCase	)
												_UpperCAmelCase									=			pa.array(OptimizedTypedSequence(__lowerCAmelCase ,		col=__lowerCAmelCase	)	)
												assert get_base_dtype(arr.type	) == pa.intaa()


@pytest.mark.parametrize('raise_exception' ,		[False, True]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  Optional[Any]:





						"""simple docstring"""



						_UpperCAmelCase									=			str(tmp_path / 'dataset-train.arrow'	)
						try:
												with ArrowWriter(path=__lowerCAmelCase	) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed


def __A    (      __lowerCAmelCase	)->  str:





						"""simple docstring"""



						_UpperCAmelCase									=			'mock://dataset-train.arrow'
						with ArrowWriter(path=__lowerCAmelCase ,		storage_options=mockfs.storage_options	) as writer:
												assert isinstance(writer._fs ,		type(__lowerCAmelCase	)	)
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({'col_1': 'foo', 'col_2': 1}	)
												writer.write({'col_1': 'bar', 'col_2': 2}	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(__lowerCAmelCase	)


def __A    (      )->  Any:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ParquetWriter(stream=__lowerCAmelCase	) as writer:
												writer.write({'col_1': 'foo', 'col_2': 1}	)
												writer.write({'col_1': 'bar', 'col_2': 2}	)
												_UpperCAmelCase					, _UpperCAmelCase									=			writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_UpperCAmelCase									=			pa.BufferReader(output.getvalue()	)
						_UpperCAmelCase									=			pq.read_table(__lowerCAmelCase	)
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}


@require_pil
@pytest.mark.parametrize('embed_local_files' ,		[False, True]	)
def __A    (      __lowerCAmelCase ,		__lowerCAmelCase	)->  str:





						"""simple docstring"""



						import PIL.Image

						_UpperCAmelCase									=			str(tmp_path / 'test_image_rgb.jpg'	)
						PIL.Image.fromarray(np.zeros((5, 5) ,		dtype=np.uinta	)	).save(__lowerCAmelCase ,		format='png'	)
						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ParquetWriter(
						    stream=__lowerCAmelCase ,		features=Features({'image': Image()}	) ,		embed_local_files=__lowerCAmelCase	) as writer:
												writer.write({'image': image_path}	)
												writer.finalize()
						_UpperCAmelCase									=			pa.BufferReader(output.getvalue()	)
						_UpperCAmelCase									=			pq.read_table(__lowerCAmelCase	)
						_UpperCAmelCase									=			pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out['image'][0]['path'] ,		__lowerCAmelCase	)
												with open(__lowerCAmelCase ,		'rb'	) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None


def __A    (      )->  Optional[Any]:





						"""simple docstring"""



						_UpperCAmelCase									=			pa.schema([pa.field('col_1' ,		pa.string() ,		nullable=__lowerCAmelCase	)]	)
						_UpperCAmelCase									=			pa.BufferOutputStream()
						with ArrowWriter(stream=__lowerCAmelCase	) as writer:
												writer._build_writer(inferred_schema=__lowerCAmelCase	)
												assert writer._schema == pa.schema([pa.field('col_1' ,		pa.string()	)]	)




"""simple docstring"""







import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)


class       SCREAMING_SNAKE_CASE__				(		lowercase				):
       """simple docstring"""


       def __init__(						self		,     snake_case__=0.01		,     snake_case__=1_000						):






         """simple docstring"""





         lowerCAmelCase     :     List[Any]								=							p_stop
         lowerCAmelCase     :     Optional[Any]								=							max_length







       def __iter__(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[Any]								=							0
         lowerCAmelCase     :     Tuple								=							False
         while not stop and count < self.max_length:
           yield count
           count += 1
           lowerCAmelCase     :     Dict								=							random.random() < self.p_stop


class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       def        lowercase__							(						self		,     snake_case__		,     snake_case__		,     snake_case__=False		,     snake_case__=True						):






         """simple docstring"""





         lowerCAmelCase     :     Dict								=							[
             BatchSamplerShard(snake_case__		,     2		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)
             for i in range(2						)
         ]
         lowerCAmelCase     :     Any								=							[list(snake_case__						) for batch_sampler_shard in batch_sampler_shards]
         if not split_batches:
           self.assertListEqual([len(snake_case__						) for shard in batch_sampler_shards]		,     [len(snake_case__						) for e in expected]						)
         self.assertListEqual(snake_case__		,     snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							BatchSampler(range(24						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Any								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     Tuple								=							BatchSampler(range(24						)		,     batch_size=3		,     drop_last=snake_case__						)
         # Expected shouldn't change
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         # Check the shards when the dataset is a round multiple of batch size but not total batch size.
         lowerCAmelCase     :     Union[str, Any]								=							BatchSampler(range(21						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     Tuple								=							BatchSampler(range(21						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Any								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
         # num_processes batch.
         lowerCAmelCase     :     List[str]								=							BatchSampler(range(22						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Dict								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     Dict								=							BatchSampler(range(22						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[Any]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
         # num_processes batch.
         lowerCAmelCase     :     Any								=							BatchSampler(range(20						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Tuple								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     List[str]								=							BatchSampler(range(20						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         # Check the shards when the dataset is very small.
         lowerCAmelCase     :     Dict								=							BatchSampler(range(2						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[[[0, 1, 0]], [[1, 0, 1]]]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

         lowerCAmelCase     :     Optional[Any]								=							BatchSampler(range(2						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Any								=							[[], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							BatchSampler(range(24						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     int								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         lowerCAmelCase     :     Dict								=							BatchSampler(range(24						)		,     batch_size=4		,     drop_last=snake_case__						)
         # Expected shouldn't change
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size.
         lowerCAmelCase     :     Optional[int]								=							BatchSampler(range(22						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[Any]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         lowerCAmelCase     :     List[Any]								=							BatchSampler(range(22						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size or num_processes.
         lowerCAmelCase     :     Tuple								=							BatchSampler(range(21						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         lowerCAmelCase     :     Any								=							BatchSampler(range(21						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[Any]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         # Check the shards when the dataset is very small.
         lowerCAmelCase     :     Optional[Any]								=							BatchSampler(range(2						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[int]								=							[[[0, 1]], [[0, 1]]]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

         lowerCAmelCase     :     Optional[Any]								=							BatchSampler(range(2						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[int]								=							[[], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[int]								=							BatchSampler(range(24						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Any								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     Optional[Any]								=							BatchSampler(range(24						)		,     batch_size=3		,     drop_last=snake_case__						)
         # Expected shouldn't change
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is a round multiple of batch size but not total batch size.
         lowerCAmelCase     :     Dict								=							BatchSampler(range(21						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Dict								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     int								=							BatchSampler(range(21						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size but has a multiple of
         # num_processes batch.
         lowerCAmelCase     :     str								=							BatchSampler(range(22						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     List[str]								=							BatchSampler(range(22						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
         # num_processes batch.
         lowerCAmelCase     :     str								=							BatchSampler(range(20						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     Any								=							BatchSampler(range(20						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[
             [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
             [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is very small.
         lowerCAmelCase     :     Optional[int]								=							BatchSampler(range(2						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[[[0, 1]], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     str								=							BatchSampler(range(2						)		,     batch_size=3		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Optional[Any]								=							[[], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     even_batches=snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     str								=							BatchSampler(range(24						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Dict								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     List[Any]								=							BatchSampler(range(24						)		,     batch_size=4		,     drop_last=snake_case__						)
         # Expected shouldn't change
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size.
         lowerCAmelCase     :     Tuple								=							BatchSampler(range(22						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     int								=							BatchSampler(range(22						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     str								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is not a round multiple of batch size or num_processes.
         lowerCAmelCase     :     int								=							BatchSampler(range(21						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     str								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     str								=							BatchSampler(range(21						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     Union[str, Any]								=							[
             [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
             [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
         ]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         # Check the shards when the dataset is very small.
         lowerCAmelCase     :     Optional[Any]								=							BatchSampler(range(2						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     int								=							[[[0, 1]], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

         lowerCAmelCase     :     Dict								=							BatchSampler(range(2						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[str]								=							[[], []]
         self.check_batch_sampler_shards(snake_case__		,     snake_case__		,     split_batches=snake_case__		,     even_batches=snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
         lowerCAmelCase     :     Tuple								=							[BatchSamplerShard(snake_case__		,     2		,     snake_case__		,     even_batches=snake_case__						) for i in range(2						)]

         self.assertEqual(len(batch_sampler_shards[0]						)		,     3						)
         self.assertEqual(len(batch_sampler_shards[1]						)		,     2						)

         self.assertListEqual(list(batch_sampler_shards[0]						)		,     [[0, 1, 2], [5, 6, 7, 8], [12, 13]]						)
         self.assertListEqual(list(batch_sampler_shards[1]						)		,     [[3, 4], [9, 10, 11]]						)

       def        lowercase__							(						self		,     snake_case__		,     snake_case__		,     snake_case__		,     snake_case__=False		,     snake_case__=2		,     snake_case__=False						):






         """simple docstring"""





         random.seed(snake_case__						)
         lowerCAmelCase     :     List[str]								=							list(snake_case__						)

         lowerCAmelCase     :     Optional[int]								=							[
             IterableDatasetShard(
                 snake_case__		,     batch_size=snake_case__		,     drop_last=snake_case__		,     num_processes=snake_case__		,     process_index=snake_case__		,     split_batches=snake_case__		,     )
             for i in range(snake_case__						)
         ]
         lowerCAmelCase     :     str								=							[]
         for iterable_dataset_shard in iterable_dataset_shards:
           # Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
           random.seed(snake_case__						)
           iterable_dataset_lists.append(list(snake_case__						)						)

         lowerCAmelCase     :     List[Any]								=							batch_size // num_processes if split_batches else batch_size
         # All iterable dataset shard should have the same length, a round multiple of shard_batch_size
         lowerCAmelCase     :     Tuple								=							iterable_dataset_lists[0]
         for l in iterable_dataset_lists[1:]:
           self.assertEqual(len(snake_case__						)		,     len(snake_case__						)						)
           self.assertTrue(len(snake_case__						) % shard_batch_size == 0						)

         lowerCAmelCase     :     List[Any]								=							[]
         for idx in range(0		,     len(snake_case__						)		,     snake_case__						):
           for l in iterable_dataset_lists:
             observed += l[idx : idx + shard_batch_size]

         if not drop_last:
           while len(snake_case__						) < len(snake_case__						):
             reference += reference
         self.assertListEqual(snake_case__		,     reference[: len(snake_case__						)]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							42
         lowerCAmelCase     :     Tuple								=							RandomIterableDataset()

         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)

         # Edge case with a very small dataset
         lowerCAmelCase     :     List[str]								=							RandomIterableDataset(max_length=2						)

         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)
         self.check_iterable_dataset_shards(snake_case__		,     snake_case__		,     batch_size=4		,     drop_last=snake_case__		,     split_batches=snake_case__						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							BatchSampler(range(16						)		,     batch_size=4		,     drop_last=snake_case__						)
         lowerCAmelCase     :     List[Any]								=							SkipBatchSampler(snake_case__		,     2						)
         self.assertListEqual(list(snake_case__						)		,     [[8, 9, 10, 11], [12, 13, 14, 15]]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							SkipDataLoader(list(range(16						)						)		,     batch_size=4		,     skip_batches=2						)
         self.assertListEqual([t.tolist() for t in dataloader]		,     [[8, 9, 10, 11], [12, 13, 14, 15]]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     int								=							DataLoader(list(range(16						)						)		,     batch_size=4						)
         lowerCAmelCase     :     Optional[int]								=							skip_first_batches(snake_case__		,     num_batches=2						)
         self.assertListEqual([t.tolist() for t in new_dataloader]		,     [[8, 9, 10, 11], [12, 13, 14, 15]]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							DataLoaderShard(list(range(16						)						)		,     batch_size=4						)
         for idx, _ in enumerate(snake_case__						):
           self.assertEqual(dataloader.end_of_dataloader		,     idx == 3						)

         # Test it also works on the second iteration
         for idx, _ in enumerate(snake_case__						):
           self.assertEqual(dataloader.end_of_dataloader		,     idx == 3						)







       def        lowercase__							(						self						):






         """simple docstring"""





         Accelerator()
         lowerCAmelCase     :     Dict								=							DataLoaderDispatcher(range(16						)		,     batch_size=4						)
         for idx, _ in enumerate(snake_case__						):
           self.assertEqual(dataloader.end_of_dataloader		,     idx == 3						)

         # Test it also works on the second iteration
         for idx, _ in enumerate(snake_case__						):
           self.assertEqual(dataloader.end_of_dataloader		,     idx == 3						)





