
'''simple docstring'''







import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


__lowerCAmelCase         =    5_0_0_0_0
__lowerCAmelCase         =    5_0_0_0

__lowerCAmelCase   ,				__lowerCAmelCase         =    os.path.split(__file__)
__lowerCAmelCase         =    os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))





@get_duration
def 					UpperCAmelCase_		(__a      :    datasets.Dataset ,	__a      :    Tuple ):






   """simple docstring"""

   for i in range(__a ):
      _a				:	Optional[int]			=   dataset[i]





@get_duration
def 					UpperCAmelCase_		(__a      :    datasets.Dataset ,	__a      :    Optional[Any] ,	__a      :    Optional[Any] ):






   """simple docstring"""

   for i in range(0 ,	len(__a ) ,	__a ):
      _a				:	Tuple			=   dataset[i : i + batch_size]





@get_duration
def 					UpperCAmelCase_		(__a      :    datasets.Dataset ,	__a      :    Optional[int] ,	__a      :    Tuple ):






   """simple docstring"""

   with dataset.formatted_as(type=__a ):
      for i in range(__a ):
         _a				:	Tuple			=   dataset[i]





@get_duration
def 					UpperCAmelCase_		(__a      :    datasets.Dataset ,	__a      :    str ,	__a      :    Union[str, Any] ,	__a      :    Union[str, Any] ):






   """simple docstring"""

   with dataset.formatted_as(type=__a ):
      for i in range(0 ,	__a ,	__a ):
         _a				:	List[str]			=   dataset[i : i + batch_size]





def 					UpperCAmelCase_		():






   """simple docstring"""

   _a				:	Union[str, Any]			=   {'num examples': SPEED_TEST_N_EXAMPLES}
   _a				:	Optional[int]			=   [
       (read, {'length': SMALL_TEST}),
       (read, {'length': SPEED_TEST_N_EXAMPLES}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0_0}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0_0_0}),
       (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
       (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_0}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_0_0_0}),
   ]

   _a				:	List[Any]			=   [
       (read, {'length': SMALL_TEST}),
       (read, {'length': SPEED_TEST_N_EXAMPLES}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0_0}),
       (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1_0_0_0}),
       (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_0}),
       (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1_0_0_0}),
   ]
   with tempfile.TemporaryDirectory() as tmp_dir:
      print('generating dataset' )
      _a				:	str			=   datasets.Features(
          {'list': datasets.Sequence(datasets.Value('float32' ) ), 'numbers': datasets.Value('float32' )} )
      _a				:	int			=   generate_example_dataset(
          os.path.join(__a ,	'dataset.arrow' ) ,	__a ,	num_examples=__a ,	seq_shapes={'list': (1_0_0,)} ,	)
      print('first set of iterations' )
      for func, kwargs in functions:
         print(func.__name__ ,	str(__a ) )
         _a				:	List[str]			=   func(__a ,	**__a )

      print('shuffling dataset' )
      _a				:	Dict			=   dataset.shuffle()
      print('Second set of iterations (after shuffling' )
      for func, kwargs in functions_shuffled:
         print('shuffled ' ,	func.__name__ ,	str(__a ) )
         _a				:	Dict			=   func(
             __a ,	**__a )

   with open(__a ,	'wb' ) as f:
      f.write(json.dumps(__a ).encode('utf-8' ) )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_iterating()





'''simple docstring'''







import os

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home


__lowerCAmelCase         =    HUGGINGFACE_HUB_CACHE


__lowerCAmelCase         =    """config.json"""
__lowerCAmelCase         =    """diffusion_pytorch_model.bin"""
__lowerCAmelCase         =    """diffusion_flax_model.msgpack"""
__lowerCAmelCase         =    """model.onnx"""
__lowerCAmelCase         =    """diffusion_pytorch_model.safetensors"""
__lowerCAmelCase         =    """weights.pb"""
__lowerCAmelCase         =    """https://huggingface.co"""
__lowerCAmelCase         =    default_cache_path
__lowerCAmelCase         =    """diffusers_modules"""
__lowerCAmelCase         =    os.getenv("""HF_MODULES_CACHE""", os.path.join(hf_cache_home, """modules"""))
__lowerCAmelCase         =    ["""fp16""", """non-ema"""]
__lowerCAmelCase         =    """.self_attn"""



