'''simple docstring'''




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__UpperCAmelCase     							=datasets.utils.logging.get_logger(__name__)




@dataclass
class     a__      (   datasets.BuilderConfig ):
     lowerCamelCase  :     Optional[datasets.Features]							=None
     lowerCamelCase  :     str							="utf-8"
     lowerCamelCase  :     Optional[str]							=None
     lowerCamelCase  :     Optional[str]							=None
     lowerCamelCase  :     bool							=True  # deprecated
     lowerCamelCase  :     Optional[int]							=None  # deprecated
     lowerCamelCase  :     int							=1_0 << 2_0  # 10MB
     lowerCamelCase  :     Optional[bool]							=None




class     a__      (   datasets.ArrowBasedBuilder ):
     lowerCamelCase  :     int							=JsonConfig






     def    SCREAMING_SNAKE_CASE__   ( self		:		Any    ):





         """simple docstring"""



         if self.config.block_size is not None:
             logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''    )
             __lowerCamelCase		=  self.config.block_size
         if self.config.use_threads is not True:
             logger.warning(
                 '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''    )
         if self.config.newlines_in_values is not None:
             raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''    )
         return datasets.DatasetInfo(features=self.config.features    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Any  ,				a		:		Union[str, Any]    ):





         """simple docstring"""



         if not self.config.data_files:
             raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
         __lowerCamelCase		=  dl_manager.download_and_extract(self.config.data_files    )
         if isinstance(_SCREAMING_SNAKE_CASE  ,				(str, list, tuple)    ):
             __lowerCamelCase		=  data_files
             if isinstance(_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE    ):
                 __lowerCamelCase		=  [files]
             __lowerCamelCase		=  [dl_manager.iter_files(_SCREAMING_SNAKE_CASE    ) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,				gen_kwargs={'''files''': files}    )]
         __lowerCamelCase		=  []
         for split_name, files in data_files.items():
             if isinstance(_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE    ):
                 __lowerCamelCase		=  [files]
             __lowerCamelCase		=  [dl_manager.iter_files(_SCREAMING_SNAKE_CASE    ) for file in files]
             splits.append(datasets.SplitGenerator(name=_SCREAMING_SNAKE_CASE  ,				gen_kwargs={'''files''': files}    )    )
         return splits






     def    SCREAMING_SNAKE_CASE__   ( self		:		Union[str, Any]  ,				a		:		List[Any]    ):





         """simple docstring"""



         if self.config.features is not None:
             # adding missing columns
             for column_name in set(self.config.features    ) - set(pa_table.column_names    ):
                 __lowerCamelCase		=  self.config.features.arrow_schema.field(_SCREAMING_SNAKE_CASE    ).type
                 __lowerCamelCase		=  pa_table.append_column(_SCREAMING_SNAKE_CASE  ,				pa.array([None] * len(_SCREAMING_SNAKE_CASE    )  ,				type=_SCREAMING_SNAKE_CASE    )    )
             # more expensive cast to support nested structures with keys in a different order
             # allows str <-> int/float or str to Audio for example
             __lowerCamelCase		=  table_cast(_SCREAMING_SNAKE_CASE  ,				self.config.features.arrow_schema    )
         return pa_table






     def    SCREAMING_SNAKE_CASE__   ( self		:		Any  ,				a		:		str    ):





         """simple docstring"""



         for file_idx, file in enumerate(itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE    )    ):
             # If the file is one json object and if we need to look at the list of items in one specific field
             if self.config.field is not None:
                 with open(_SCREAMING_SNAKE_CASE  ,				encoding=self.config.encoding  ,				errors=self.config.encoding_errors    ) as f:
                     __lowerCamelCase		=  json.load(_SCREAMING_SNAKE_CASE    )

                 # We keep only the field we are interested in
                 __lowerCamelCase		=  dataset[self.config.field]

                 # We accept two format: a list of dicts or a dict of lists
                 if isinstance(_SCREAMING_SNAKE_CASE  ,				(list, tuple)    ):
                     __lowerCamelCase		=  set().union(*[row.keys() for row in dataset]    )
                     __lowerCamelCase		=  {col: [row.get(_SCREAMING_SNAKE_CASE    ) for row in dataset] for col in keys}
                 else:
                     __lowerCamelCase		=  dataset
                 __lowerCamelCase		=  pa.Table.from_pydict(_SCREAMING_SNAKE_CASE    )
                 yield file_idx, self._cast_table(_SCREAMING_SNAKE_CASE    )

             # If the file has one json object per line
             else:
                 with open(_SCREAMING_SNAKE_CASE  ,				'''rb'''    ) as f:
                     __lowerCamelCase		=  0
                     # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                     # Set a default minimum value of 16kB if the chunk size is really small
                     __lowerCamelCase		=  max(self.config.chunksize // 32  ,				16 << 10    )
                     __lowerCamelCase		=  (
                         self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                     )
                     while True:
                         __lowerCamelCase		=  f.read(self.config.chunksize    )
                         if not batch:
                             break
                         # Finish current line
                         try:
                             batch += f.readline()
                         except (AttributeError, io.UnsupportedOperation):
                             batch += readline(_SCREAMING_SNAKE_CASE    )
                         # PyArrow only accepts utf-8 encoded bytes
                         if self.config.encoding != "utf-8":
                             __lowerCamelCase		=  batch.decode(self.config.encoding  ,				errors=_SCREAMING_SNAKE_CASE    ).encode('''utf-8'''    )
                         try:
                             while True:
                                 try:
                                     __lowerCamelCase		=  paj.read_json(
                                         io.BytesIO(_SCREAMING_SNAKE_CASE    )  ,				read_options=paj.ReadOptions(block_size=_SCREAMING_SNAKE_CASE    )    )
                                     break
                                 except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                     if (
                                         isinstance(_SCREAMING_SNAKE_CASE  ,				pa.ArrowInvalid    )
                                         and "straddling" not in str(_SCREAMING_SNAKE_CASE    )
                                         or block_size > len(_SCREAMING_SNAKE_CASE    )
                                     ):
                                         raise
                                     else:
                                         # Increase the block size in case it was too small.
                                         # The block size will be reset for the next file.
                                         logger.debug(
                                             f"""Batch of {len(_SCREAMING_SNAKE_CASE    )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""    )
                                         block_size *= 2
                         except pa.ArrowInvalid as e:
                             try:
                                 with open(
                                     _SCREAMING_SNAKE_CASE  ,				encoding=self.config.encoding  ,				errors=self.config.encoding_errors    ) as f:
                                     __lowerCamelCase		=  json.load(_SCREAMING_SNAKE_CASE    )
                             except json.JSONDecodeError:
                                 logger.error(f"""Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE    )}: {e}"""    )
                                 raise e
                             # If possible, parse the file as a list of json objects and exit the loop
                             if isinstance(_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE    ):  # list is the only sequence type supported in JSON
                                 try:
                                     __lowerCamelCase		=  set().union(*[row.keys() for row in dataset]    )
                                     __lowerCamelCase		=  {col: [row.get(_SCREAMING_SNAKE_CASE    ) for row in dataset] for col in keys}
                                     __lowerCamelCase		=  pa.Table.from_pydict(_SCREAMING_SNAKE_CASE    )
                                 except (pa.ArrowInvalid, AttributeError) as e:
                                     logger.error(f"""Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE    )}: {e}"""    )
                                     raise ValueError(f"""Not able to read records in the JSON file at {file}."""    ) from None
                                 yield file_idx, self._cast_table(_SCREAMING_SNAKE_CASE    )
                                 break
                             else:
                                 logger.error(f"""Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE    )}: {e}"""    )
                                 raise ValueError(
                                     f"""Not able to read records in the JSON file at {file}. """
                                     f"""You should probably indicate the field of the JSON file containing your records. """
                                     f"""This JSON file contain the following fields: {str(list(dataset.keys()    )    )}. """
                                     f"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """    ) from None
                                               # Uncomment for debugging (will print the Arrow table size and elements)
                                               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                         yield (file_idx, batch_idx), self._cast_table(_SCREAMING_SNAKE_CASE    )
                         batch_idx += 1





"""simple docstring"""
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser

def       _snake_case						(						):
 UpperCAmelCase	:					List[str]	=	ArgumentParser("""Accelerate CLI tool"""			,		usage="""accelerate <command> [<args>]"""			,		allow_abbrev=UpperCamelCase						)
 UpperCAmelCase	:					Dict	=	parser.add_subparsers(help="""accelerate command helpers"""						)

 # Register commands
 get_config_parser(subparsers=UpperCamelCase						)
 env_command_parser(subparsers=UpperCamelCase						)
 launch_command_parser(subparsers=UpperCamelCase						)
 tpu_command_parser(subparsers=UpperCamelCase						)
 test_command_parser(subparsers=UpperCamelCase						)

 # Let's go
 UpperCAmelCase	:					Optional[int]	=	parser.parse_args()

 if not hasattr(UpperCamelCase			,		"""func"""						):
  parser.print_help()
  exit(1						)

 # Run
 args.func(UpperCamelCase						)


if __name__ == "__main__":
      main()
