

import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_UpperCAmelCase		:   Optional[int]										=       5_0000
_UpperCAmelCase		:   Dict										=       5000

_UpperCAmelCase				,						_UpperCAmelCase		:   Optional[int]										=       os.path.split(__file__)
_UpperCAmelCase		:   List[str]										=       os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							for i in range(UpperCamelCase__      ):
														snake_case_							=    dataset[i]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							for i in range(0   ,							len(UpperCamelCase__      )   ,							UpperCamelCase__      ):
														snake_case_							=    dataset[i : i + batch_size]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							with dataset.formatted_as(type=UpperCamelCase__      ):
														for i in range(UpperCamelCase__      ):
																					snake_case_							=    dataset[i]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							with dataset.formatted_as(type=UpperCamelCase__      ):
														for i in range(0   ,							UpperCamelCase__   ,							UpperCamelCase__      ):
																					snake_case_							=    dataset[i : i + batch_size]





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    {'num examples': SPEED_TEST_N_EXAMPLES}
							snake_case_							=    [
							    (read, {'length': SMALL_TEST}),
							    (read, {'length': SPEED_TEST_N_EXAMPLES}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
							    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
							]

							snake_case_							=    [
							    (read, {'length': SMALL_TEST}),
							    (read, {'length': SPEED_TEST_N_EXAMPLES}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
							    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
							]
							with tempfile.TemporaryDirectory() as tmp_dir:
														print('generating dataset'      )
														snake_case_							=    datasets.Features(
														    {'list': datasets.Sequence(datasets.Value('float32'      )      ), 'numbers': datasets.Value('float32'      )}      )
														snake_case_							=    generate_example_dataset(
														    os.path.join(UpperCamelCase__   ,							'dataset.arrow'      )   ,							UpperCamelCase__   ,							num_examples=UpperCamelCase__   ,							seq_shapes={'list': (100,)}   ,							)
														print('first set of iterations'      )
														for func, kwargs in functions:
																					print(func.__name__   ,							str(UpperCamelCase__      )      )
																					snake_case_							=    func(UpperCamelCase__   ,							**UpperCamelCase__      )

														print('shuffling dataset'      )
														snake_case_							=    dataset.shuffle()
														print('Second set of iterations (after shuffling'      )
														for func, kwargs in functions_shuffled:
																					print('shuffled '   ,							func.__name__   ,							str(UpperCamelCase__      )      )
																					snake_case_							=    func(
																					    UpperCamelCase__   ,							**UpperCamelCase__      )

							with open(UpperCamelCase__   ,							'wb'      ) as f:
														f.write(json.dumps(UpperCamelCase__      ).encode('utf-8'      )      )


if __name__ == "__main__":  # useful to run the profiler
					benchmark_iterating()







import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    'files'   ,							[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    tmp_path_factory.mktemp('dset_infos_dir'      )
							if "full:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write('---\ndataset_info:\n  dataset_size: 42\n---'      )
							if "empty:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write(''      )
    # we want to support dataset_infos.json for backward compatibility
							if "dataset_infos.json" in files:
														with open(dataset_infos_dir / 'dataset_infos.json'   ,							'w'      ) as f:
																					f.write('{"default": {"dataset_size": 42}}'      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )
							assert dataset_infos
							assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    'dataset_info'   ,							[
        DatasetInfo(),
        DatasetInfo(
            description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_info.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfo.from_directory(UpperCamelCase__      )
							assert dataset_info == reloaded
							assert os.path.exists(os.path.join(UpperCamelCase__   ,							'dataset_info.json'      )      )





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo(
							    description='foo'   ,							citation='bar'   ,							homepage='https://foo.bar'   ,							license='CC0'   ,							features=Features({'a': Value('int32'      )}      )   ,							post_processed={}   ,							supervised_keys=()   ,							task_templates=[]   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train', 'num_examples': 42}]   ,							download_checksums={}   ,							download_size=1337   ,							post_processing_size=442   ,							dataset_size=1234   ,							size_in_bytes=1337 + 442 + 1234   ,							)
							snake_case_							=    dataset_info._to_yaml_dict()
							assert sorted(UpperCamelCase__      ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML      )
							for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
														assert key in dataset_info_yaml_dict
														assert isinstance(dataset_info_yaml_dict[key]   ,							(list, dict, int, str)      )
							snake_case_							=    yaml.safe_dump(UpperCamelCase__      )
							snake_case_							=    yaml.safe_load(UpperCamelCase__      )
							assert dataset_info_yaml_dict == reloaded





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo()
							snake_case_							=    dataset_info._to_yaml_dict()
							assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    'dataset_infos_dict'   ,							[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()}      ),
        DatasetInfosDict({'my_config_name': DatasetInfo()}      ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							)
            }      ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42      ),
                'v2': DatasetInfo(dataset_size=1337      ),
            }      ),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_infos_dict.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )

							# the config_name of the dataset_infos_dict take over the attribute
							for config_name, dataset_info in dataset_infos_dict.items():
														snake_case_							=    config_name
														# the yaml representation doesn't include fields like description or citation
														# so we just test that we can recover what we can from the yaml
														snake_case_							=    DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()      )
							assert dataset_infos_dict == reloaded

							if dataset_infos_dict:
														assert os.path.exists(os.path.join(UpperCamelCase__   ,							'README.md'      )      )




