


"""simple docstring"""






import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					def __init__(       self    : Union[str, Any] ,       UpperCamelCase    : NestedDataStructureLike[PathLike] ,       UpperCamelCase    : Optional[NamedSplit] = None ,       UpperCamelCase    : Optional[Features] = None ,       UpperCamelCase    : str = None ,       UpperCamelCase    : bool = False ,       UpperCamelCase    : bool = False ,       UpperCamelCase    : Optional[str] = None ,       UpperCamelCase    : Optional[int] = None ,       **UpperCamelCase    : Union[str, Any] ,       ):
												'''simple docstring'''






												super().__init__(
												    UpperCamelCase ,       split=UpperCamelCase ,       features=UpperCamelCase ,       cache_dir=UpperCamelCase ,       keep_in_memory=UpperCamelCase ,       streaming=UpperCamelCase ,       num_proc=UpperCamelCase ,       **UpperCamelCase ,       )
												__UpperCAmelCase   :						Optional[Any]      							=					field
												__UpperCAmelCase   :						Union[str, Any]      							=					path_or_paths if isinstance(UpperCamelCase ,       UpperCamelCase  ) else {self.split: path_or_paths}
												__UpperCAmelCase   :						Any      							=					Json(
												    cache_dir=UpperCamelCase ,       data_files=UpperCamelCase ,       features=UpperCamelCase ,       field=UpperCamelCase ,       **UpperCamelCase ,       )






					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												if self.streaming:
																			__UpperCAmelCase   :						Any      							=					self.builder.as_streaming_dataset(split=self.split  )
												# Build regular (map-style) dataset
												else:
																			__UpperCAmelCase   :						Dict      							=					None
																			__UpperCAmelCase   :						List[str]      							=					None
																			__UpperCAmelCase   :						int      							=					None
																			__UpperCAmelCase   :						List[Any]      							=					None

																			self.builder.download_and_prepare(
																			    download_config=UpperCamelCase ,       download_mode=UpperCamelCase ,       verification_mode=UpperCamelCase ,       base_path=UpperCamelCase ,       num_proc=self.num_proc ,       )
																			__UpperCAmelCase   :						Union[str, Any]      							=					self.builder.as_dataset(
																			    split=self.split ,       verification_mode=UpperCamelCase ,       in_memory=self.keep_in_memory  )
												return dataset






class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : List[Any] ,       UpperCamelCase    : Dataset ,       UpperCamelCase    : Union[PathLike, BinaryIO] ,       UpperCamelCase    : Optional[int] = None ,       UpperCamelCase    : Optional[int] = None ,       **UpperCamelCase    : List[str] ,       ):
												'''simple docstring'''






												if num_proc is not None and num_proc <= 0:
																			raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''  )

												__UpperCAmelCase   :						int      							=					dataset
												__UpperCAmelCase   :						int      							=					path_or_buf
												__UpperCAmelCase   :						Dict      							=					batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
												__UpperCAmelCase   :						Union[str, Any]      							=					num_proc
												__UpperCAmelCase   :						Any      							=					"""utf-8"""
												__UpperCAmelCase   :						Optional[int]      							=					to_json_kwargs






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					self.to_json_kwargs.pop("""path_or_buf""" ,       UpperCamelCase  )
												__UpperCAmelCase   :						List[Any]      							=					self.to_json_kwargs.pop("""orient""" ,       """records"""  )
												__UpperCAmelCase   :						Union[str, Any]      							=					self.to_json_kwargs.pop("""lines""" ,       True if orient == """records""" else False  )
												__UpperCAmelCase   :						Dict      							=					self.to_json_kwargs.pop("""index""" ,       False if orient in ["""split""", """table"""] else True  )
												__UpperCAmelCase   :						Optional[int]      							=					self.to_json_kwargs.pop("""compression""" ,       UpperCamelCase  )

												if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																			raise NotImplementedError(f'''`datasets` currently does not support {compression} compression'''  )

												if isinstance(self.path_or_buf ,       (str, bytes, os.PathLike)  ):
																			with fsspec.open(self.path_or_buf ,       """wb""" ,       compression=UpperCamelCase  ) as buffer:
																										__UpperCAmelCase   :						Tuple      							=					self._write(file_obj=UpperCamelCase ,       orient=UpperCamelCase ,       lines=UpperCamelCase ,       index=UpperCamelCase ,       **self.to_json_kwargs  )
												else:
																			if compression:
																										raise NotImplementedError(
																										    f'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
																										    """ was passed. Please provide a local path instead."""  )
																			__UpperCAmelCase   :						int      							=					self._write(
																			    file_obj=self.path_or_buf ,       orient=UpperCamelCase ,       lines=UpperCamelCase ,       index=UpperCamelCase ,       **self.to_json_kwargs  )
												return written






					def lowerCamelCase__							(       self    : Dict ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase							,__UpperCAmelCase							,__UpperCAmelCase							,__UpperCAmelCase							,__UpperCAmelCase   :						Union[str, Any]      							=					args

												__UpperCAmelCase   :						Union[str, Any]      							=					query_table(
												    table=self.dataset.data ,       key=slice(UpperCamelCase ,       offset + self.batch_size  ) ,       indices=self.dataset._indices ,       )
												__UpperCAmelCase   :						List[Any]      							=					batch.to_pandas().to_json(
												    path_or_buf=UpperCamelCase ,       orient=UpperCamelCase ,       lines=UpperCamelCase ,       index=UpperCamelCase ,       **UpperCamelCase  )
												if not json_str.endswith("""\n"""  ):
																			json_str += "\n"
												return json_str.encode(self.encoding  )






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : BinaryIO ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : Any ,       UpperCamelCase    : Dict ,       **UpperCamelCase    : Union[str, Any] ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					0

												if self.num_proc is None or self.num_proc == 1:
																			for offset in logging.tqdm(
																			    range(0 ,       len(self.dataset  ) ,       self.batch_size  ) ,       unit="""ba""" ,       disable=not logging.is_progress_bar_enabled() ,       desc="""Creating json from Arrow format""" ,       ):
																										__UpperCAmelCase   :						str      							=					self._batch_json((offset, orient, lines, index, to_json_kwargs)  )
																										written += file_obj.write(UpperCamelCase  )
												else:
																			__UpperCAmelCase							,__UpperCAmelCase   :						Union[str, Any]      							=					len(self.dataset  ), self.batch_size
																			with multiprocessing.Pool(self.num_proc  ) as pool:
																										for json_str in logging.tqdm(
																										    pool.imap(
																										        self._batch_json ,       [(offset, orient, lines, index, to_json_kwargs) for offset in range(0 ,       UpperCamelCase ,       UpperCamelCase  )] ,       ) ,       total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size ,       unit="""ba""" ,       disable=not logging.is_progress_bar_enabled() ,       desc="""Creating json from Arrow format""" ,       ):
																																	written += file_obj.write(UpperCamelCase  )

												return written




"""simple docstring"""






from importlib import import_module

from .logging import get_logger


UpperCAmelCase    :     Any   						=		get_logger(__name__)





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : List[Any] ,       UpperCamelCase    : int=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					attrs or []
												if module is not None:
																			for key in module.__dict__:
																										if key in attrs or not key.startswith("""__"""  ):
																																	setattr(self ,       UpperCamelCase ,       getattr(UpperCamelCase ,       UpperCamelCase  )  )
												__UpperCAmelCase   :						Any      							=					module._original_module if isinstance(UpperCamelCase ,       _PatchedModuleObj  ) else module






class      lowerCamelCase__   :







					"""simple docstring"""


					__a	       =							[]






					def __init__(       self    : str ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : str ,       UpperCamelCase    : List[str] ,       UpperCamelCase    : Optional[Any]=None  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					obj
												__UpperCAmelCase   :						Union[str, Any]      							=					target
												__UpperCAmelCase   :						List[str]      							=					new
												__UpperCAmelCase   :						Optional[int]      							=					target.split("""."""  )[0]
												__UpperCAmelCase   :						Tuple      							=					{}
												__UpperCAmelCase   :						Union[str, Any]      							=					attrs or []






					def __enter__(       self    : Dict  ):
												'''simple docstring'''






												*__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					self.target.split("""."""  )

												# Patch modules:
												# it's used to patch attributes of submodules like "os.path.join";
												# in this case we need to patch "os" and "os.path"

												for i in range(len(UpperCamelCase  )  ):
																			try:
																										__UpperCAmelCase   :						List[Any]      							=					import_module(""".""".join(submodules[: i + 1]  )  )
																			except ModuleNotFoundError:
																										continue
																			# We iterate over all the globals in self.obj in case we find "os" or "os.path"
																			for attr in self.obj.__dir__():
																										__UpperCAmelCase   :						List[Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																										# We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
																										# This allows to patch renamed modules like "from os import path as ospath".
																										if obj_attr is submodule or (
																										    (isinstance(UpperCamelCase ,       _PatchedModuleObj  ) and obj_attr._original_module is submodule)
																										):
																																	__UpperCAmelCase   :						Tuple      							=					obj_attr
																																	# patch at top level
																																	setattr(self.obj ,       UpperCamelCase ,       _PatchedModuleObj(UpperCamelCase ,       attrs=self.attrs  )  )
																																	__UpperCAmelCase   :						int      							=					getattr(self.obj ,       UpperCamelCase  )
																																	# construct lower levels patches
																																	for key in submodules[i + 1 :]:
																																								setattr(UpperCamelCase ,       UpperCamelCase ,       _PatchedModuleObj(getattr(UpperCamelCase ,       UpperCamelCase ,       UpperCamelCase  ) ,       attrs=self.attrs  )  )
																																								__UpperCAmelCase   :						Optional[int]      							=					getattr(UpperCamelCase ,       UpperCamelCase  )
																																	# finally set the target attribute
																																	setattr(UpperCamelCase ,       UpperCamelCase ,       self.new  )

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

												if submodules:  # if it's an attribute of a submodule like "os.path.join"
																			try:
																										__UpperCAmelCase   :						int      							=					getattr(import_module(""".""".join(UpperCamelCase  )  ) ,       UpperCamelCase  )
																			except (AttributeError, ModuleNotFoundError):
																										return
																			# We iterate over all the globals in self.obj in case we find "os.path.join"
																			for attr in self.obj.__dir__():
																										# We don't check for the name of the global, but rather if its value *is* "os.path.join".
																										# This allows to patch renamed attributes like "from os.path import join as pjoin".
																										if getattr(self.obj ,       UpperCamelCase  ) is attr_value:
																																	__UpperCAmelCase   :						Union[str, Any]      							=					getattr(self.obj ,       UpperCamelCase  )
																																	setattr(self.obj ,       UpperCamelCase ,       self.new  )
												elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
																			__UpperCAmelCase   :						str      							=					globals()["""__builtins__"""][target_attr]
																			setattr(self.obj ,       UpperCamelCase ,       self.new  )
												else:
																			raise RuntimeError(f'''Tried to patch attribute {target_attr} instead of a submodule.'''  )






					def __exit__(       self    : str ,       *UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												for attr in list(self.original  ):
																			setattr(self.obj ,       UpperCamelCase ,       self.original.pop(UpperCamelCase  )  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.__enter__()
												self._active_patches.append(self  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												try:
																			self._active_patches.remove(self  )
												except ValueError:
																			# If the patch hasn't been started this will fail
																			return None

												return self.__exit__()
