


'''simple docstring'''



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = False     ,	SCREAMING_SNAKE_CASE_ = False     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Dict:
         super().__init__(
             SCREAMING_SNAKE_CASE_     ,	split=SCREAMING_SNAKE_CASE_     ,	features=SCREAMING_SNAKE_CASE_     ,	cache_dir=SCREAMING_SNAKE_CASE_     ,	keep_in_memory=SCREAMING_SNAKE_CASE_     ,	streaming=SCREAMING_SNAKE_CASE_     ,	num_proc=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)
         __lowerCamelCase  :      Any														=						field
         __lowerCamelCase  :      Tuple														=						path_or_paths if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else {self.split: path_or_paths}
         __lowerCamelCase  :      int														=						Json(
             cache_dir=SCREAMING_SNAKE_CASE_     ,	data_files=SCREAMING_SNAKE_CASE_     ,	features=SCREAMING_SNAKE_CASE_     ,	field=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)







    def  lowercase_							(				self      )							->	List[str]:
         # Build iterable dataset
         if self.streaming:
              __lowerCamelCase  :      str														=						self.builder.as_streaming_dataset(split=self.split      )
         # Build regular (map-style) dataset
         else:
              __lowerCamelCase  :      List[Any]														=						None
              __lowerCamelCase  :      Optional[Any]														=						None
              __lowerCamelCase  :      Union[str, Any]														=						None
              __lowerCamelCase  :      Dict														=						None

              self.builder.download_and_prepare(
                  download_config=SCREAMING_SNAKE_CASE_     ,	download_mode=SCREAMING_SNAKE_CASE_     ,	verification_mode=SCREAMING_SNAKE_CASE_     ,	base_path=SCREAMING_SNAKE_CASE_     ,	num_proc=self.num_proc     ,	)
              __lowerCamelCase  :      List[str]														=						self.builder.as_dataset(
                  split=self.split     ,	verification_mode=SCREAMING_SNAKE_CASE_     ,	in_memory=self.keep_in_memory      )
         return dataset







class   UpperCAmelCase_    :





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	int:
         if num_proc is not None and num_proc <= 0:
              raise ValueError(f'num_proc {num_proc} must be an integer > 0.'      )

         __lowerCamelCase  :      Tuple														=						dataset
         __lowerCamelCase  :      Dict														=						path_or_buf
         __lowerCamelCase  :      int														=						batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         __lowerCamelCase  :      str														=						num_proc
         __lowerCamelCase  :      Union[str, Any]														=						'utf-8'
         __lowerCamelCase  :      str														=						to_json_kwargs







    def  lowercase_							(				self      )							->	int:
         __lowerCamelCase  :      Optional[Any]														=						self.to_json_kwargs.pop('path_or_buf'     ,	SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Tuple														=						self.to_json_kwargs.pop('orient'     ,	'records'      )
         __lowerCamelCase  :      List[Any]														=						self.to_json_kwargs.pop('lines'     ,	True if orient == 'records' else False      )
         __lowerCamelCase  :      List[str]														=						self.to_json_kwargs.pop('index'     ,	False if orient in ['split', 'table'] else True      )
         __lowerCamelCase  :      Tuple														=						self.to_json_kwargs.pop('compression'     ,	SCREAMING_SNAKE_CASE_      )

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
              raise NotImplementedError(f'`datasets` currently does not support {compression} compression'      )

         if isinstance(self.path_or_buf     ,	(str, bytes, os.PathLike)      ):
              with fsspec.open(self.path_or_buf     ,	'wb'     ,	compression=SCREAMING_SNAKE_CASE_      ) as buffer:
                   __lowerCamelCase  :      List[Any]														=						self._write(file_obj=SCREAMING_SNAKE_CASE_     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**self.to_json_kwargs      )
         else:
              if compression:
                   raise NotImplementedError(
                       f'The compression parameter is not supported when writing to a buffer, but compression={compression}'
                       ' was passed. Please provide a local path instead.'      )
              __lowerCamelCase  :      Dict														=						self._write(
                  file_obj=self.path_or_buf     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**self.to_json_kwargs      )
         return written







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[str]:
         __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase  :      str														=						args

         __lowerCamelCase  :      List[str]														=						query_table(
             table=self.dataset.data     ,	key=slice(SCREAMING_SNAKE_CASE_     ,	offset + self.batch_size      )     ,	indices=self.dataset._indices     ,	)
         __lowerCamelCase  :      Tuple														=						batch.to_pandas().to_json(
             path_or_buf=SCREAMING_SNAKE_CASE_     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )
         if not json_str.endswith('\n'      ):
              json_str += "\n"
         return json_str.encode(self.encoding      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	int:
         __lowerCamelCase  :      Any														=						0

         if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0     ,	len(self.dataset      )     ,	self.batch_size      )     ,	unit='ba'     ,	disable=not logging.is_progress_bar_enabled()     ,	desc='Creating json from Arrow format'     ,	):
                   __lowerCamelCase  :      Optional[int]														=						self._batch_json((offset, orient, lines, index, to_json_kwargs)      )
                   written += file_obj.write(SCREAMING_SNAKE_CASE_      )
         else:
              __lowerCamelCase							,			__lowerCamelCase  :      Optional[Any]														=						len(self.dataset      ), self.batch_size
              with multiprocessing.Pool(self.num_proc      ) as pool:
                   for json_str in logging.tqdm(
                       pool.imap(
                           self._batch_json     ,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )]     ,	)     ,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,	unit='ba'     ,	disable=not logging.is_progress_bar_enabled()     ,	desc='Creating json from Arrow format'     ,	):
                        written += file_obj.write(SCREAMING_SNAKE_CASE_      )

         return written











from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_tf_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_tf_available():
     import tensorflow as tf


a_    :  Any						  =		logging.get_logger(__name__)





@dataclass
class  lowerCamelCase__	(    UpperCAmelCase_):
     """simple docstring"""



     _A      =      [
         'no_inference',
         'no_cuda',
         'no_tpu',
         'no_speed',
         'no_memory',
         'no_env_print',
         'no_multi_process',
     ]







     def __init__(self					,      **__a     ):






           '''simple docstring'''



           for deprecated_arg in self.deprecated_args:
                 if deprecated_arg in kwargs:
                       lowerCamelCase             =		deprecated_arg[3:]
                       lowerCamelCase             =		not kwargs.pop(__a     )
                       logger.warning(
                           F"""{deprecated_arg} is depreciated. Please use --no-{positive_arg} or"""
                           F""" {positive_arg}={kwargs[positive_arg]}"""     )
           lowerCamelCase             =		kwargs.pop("tpu_name"					,      self.tpu_name     )
           lowerCamelCase             =		kwargs.pop("device_idx"					,      self.device_idx     )
           lowerCamelCase             =		kwargs.pop("eager_mode"					,      self.eager_mode     )
           lowerCamelCase             =		kwargs.pop("use_xla"					,      self.use_xla     )
           super().__init__(**__a     )

     _A      =      field(
         default=UpperCAmelCase_							,     metadata={'help': 'Name of TPU'}							,     )
     _A      =      field(
         default=0							,     metadata={'help': 'CPU / GPU device index. Defaults to 0.'}							,     )
     _A      =      field(default=UpperCAmelCase_							,     metadata={'help': 'Benchmark models in eager model.'})
     _A      =      field(
         default=UpperCAmelCase_							,     metadata={
             'help': 'Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`.'
         }							,     )







     @cached_property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           lowerCamelCase             =		None
           if self.tpu:
                 try:
                       if self.tpu_name:
                             lowerCamelCase             =		tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name     )
                       else:
                             lowerCamelCase             =		tf.distribute.cluster_resolver.TPUClusterResolver()
                 except ValueError:
                       lowerCamelCase             =		None
           return tpu







     @cached_property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           if self.is_tpu:
                 tf.config.experimental_connect_to_cluster(self._setup_tpu     )
                 tf.tpu.experimental.initialize_tpu_system(self._setup_tpu     )

                 lowerCamelCase             =		tf.distribute.TPUStrategy(self._setup_tpu     )
           else:
                 # currently no multi gpu is allowed
                 if self.is_gpu:
                       # TODO: Currently only single GPU is supported
                       tf.config.set_visible_devices(self.gpu_list[self.device_idx]					,      "GPU"     )
                       lowerCamelCase             =		tf.distribute.OneDeviceStrategy(device=F"""/gpu:{self.device_idx}"""     )
                 else:
                       tf.config.set_visible_devices([]					,      "GPU"     )  # disable GPU
                       lowerCamelCase             =		tf.distribute.OneDeviceStrategy(device=F"""/cpu:{self.device_idx}"""     )

           return strategy







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return self._setup_tpu is not None







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return self._setup_strategy







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return tf.config.list_physical_devices("GPU"     )







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           if self.cuda:
                 return len(self.gpu_list     )
           return 0







     @property
     def     _a   (self     ):






           '''simple docstring'''



           return self.n_gpu > 0