



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a	:   Tuple       			=       {
    "configuration_git": ["GIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "GitConfig", "GitVisionConfig"],
    "processing_git": ["GitProcessor"],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a	:   str       			=       [
	    "GIT_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "GitForCausalLM",
	    "GitModel",
	    "GitPreTrainedModel",
	    "GitVisionModel",
	]

if TYPE_CHECKING:
	from .configuration_git import GIT_PRETRAINED_CONFIG_ARCHIVE_MAP, GitConfig, GitVisionConfig
	from .processing_git import GitProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_git import (
		    GIT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    GitForCausalLM,
		    GitModel,
		    GitPreTrainedModel,
		    GitVisionModel,
		)

else:
	import sys

	a	:   Union[str, Any]       			=       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)











"""simple docstring"""
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


A:   Any								=		datasets.utils.logging.get_logger(__name__)



@dataclass
class       SCREAMING_SNAKE_CASE__  (					datasets.BuilderConfig ):
   __lowerCAmelCase  :   Optional[datasets.Features]						=						None
   __lowerCAmelCase  :   str						=						"utf-8"
   __lowerCAmelCase  :   Optional[str]						=						None
   __lowerCAmelCase  :   Optional[str]						=						None
   __lowerCAmelCase  :   bool						=						True  # deprecated
   __lowerCAmelCase  :   Optional[int]						=						None  # deprecated
   __lowerCAmelCase  :   int						=						10 << 20  # 10MB
   __lowerCAmelCase  :   Optional[bool]						=						None



class       SCREAMING_SNAKE_CASE__  (					datasets.ArrowBasedBuilder ):
   __lowerCAmelCase  :   int						=						JsonConfig

   def 	SCREAMING_SNAKE_CASE     (   self						)  ->      Any:





    '''simple docstring'''




    if self.config.block_size is not None:
     logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""						)
     UpperCAmelCase	:					Any	=	self.config.block_size
    if self.config.use_threads is not True:
     logger.warning(
         """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""						)
    if self.config.newlines_in_values is not None:
     raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""						)
    return datasets.DatasetInfo(features=self.config.features						)

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      Tuple:





    '''simple docstring'''




    if not self.config.data_files:
     raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"						)
    UpperCAmelCase	:					List[Any]	=	dl_manager.download_and_extract(self.config.data_files						)
    if isinstance(_SCREAMING_SNAKE_CASE      ,     (str, list, tuple)						):
     UpperCAmelCase	:					int	=	data_files
     if isinstance(_SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE						):
      UpperCAmelCase	:					Tuple	=	[files]
     UpperCAmelCase	:					List[str]	=	[dl_manager.iter_files(_SCREAMING_SNAKE_CASE						) for file in files]
     return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,     gen_kwargs={"""files""": files}						)]
    UpperCAmelCase	:					List[str]	=	[]
    for split_name, files in data_files.items():
     if isinstance(_SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE						):
      UpperCAmelCase	:					Dict	=	[files]
     UpperCAmelCase	:					str	=	[dl_manager.iter_files(_SCREAMING_SNAKE_CASE						) for file in files]
     splits.append(datasets.SplitGenerator(name=_SCREAMING_SNAKE_CASE      ,     gen_kwargs={"""files""": files}						)						)
    return splits

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      pa.Table:





    '''simple docstring'''




    if self.config.features is not None:
     # adding missing columns
     for column_name in set(self.config.features						) - set(pa_table.column_names						):
      UpperCAmelCase	:					Tuple	=	self.config.features.arrow_schema.field(_SCREAMING_SNAKE_CASE						).type
      UpperCAmelCase	:					int	=	pa_table.append_column(_SCREAMING_SNAKE_CASE      ,     pa.array([None] * len(_SCREAMING_SNAKE_CASE						)      ,     type=_SCREAMING_SNAKE_CASE						)						)
     # more expensive cast to support nested structures with keys in a different order
     # allows str <-> int/float or str to Audio for example
     UpperCAmelCase	:					Any	=	table_cast(_SCREAMING_SNAKE_CASE      ,     self.config.features.arrow_schema						)
    return pa_table




   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE						)  ->      Optional[Any]:





    '''simple docstring'''




    for file_idx, file in enumerate(itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE						)						):
     # If the file is one json object and if we need to look at the list of items in one specific field
     if self.config.field is not None:
      with open(_SCREAMING_SNAKE_CASE      ,     encoding=self.config.encoding      ,     errors=self.config.encoding_errors						) as f:
       UpperCAmelCase	:					Union[str, Any]	=	json.load(_SCREAMING_SNAKE_CASE						)

      # We keep only the field we are interested in
      UpperCAmelCase	:					Dict	=	dataset[self.config.field]

      # We accept two format: a list of dicts or a dict of lists
      if isinstance(_SCREAMING_SNAKE_CASE      ,     (list, tuple)						):
       UpperCAmelCase	:					Optional[Any]	=	set().union(*[row.keys() for row in dataset]						)
       UpperCAmelCase	:					Union[str, Any]	=	{col: [row.get(_SCREAMING_SNAKE_CASE						) for row in dataset] for col in keys}
      else:
       UpperCAmelCase	:					Optional[Any]	=	dataset
      UpperCAmelCase	:					List[str]	=	pa.Table.from_pydict(_SCREAMING_SNAKE_CASE						)
      yield file_idx, self._cast_table(_SCREAMING_SNAKE_CASE						)

     # If the file has one json object per line
     else:
      with open(_SCREAMING_SNAKE_CASE      ,     """rb"""						) as f:
       UpperCAmelCase	:					Union[str, Any]	=	0
       # Use block_size equal to the chunk size divided by 32 to leverage multithreading
       # Set a default minimum value of 16kB if the chunk size is really small
       UpperCAmelCase	:					int	=	max(self.config.chunksize // 32      ,     16 << 10						)
       UpperCAmelCase	:					Dict	=	(
           self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
       )
       while True:
        UpperCAmelCase	:					Tuple	=	f.read(self.config.chunksize						)
        if not batch:
         break
        # Finish current line
        try:
         batch += f.readline()
        except (AttributeError, io.UnsupportedOperation):
         batch += readline(_SCREAMING_SNAKE_CASE						)
        # PyArrow only accepts utf-8 encoded bytes
        if self.config.encoding != "utf-8":
         UpperCAmelCase	:					Tuple	=	batch.decode(self.config.encoding      ,     errors=_SCREAMING_SNAKE_CASE						).encode("""utf-8"""						)
        try:
         while True:
          try:
           UpperCAmelCase	:					int	=	paj.read_json(
               io.BytesIO(_SCREAMING_SNAKE_CASE						)      ,     read_options=paj.ReadOptions(block_size=_SCREAMING_SNAKE_CASE						)						)
           break
          except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
           if (
               isinstance(_SCREAMING_SNAKE_CASE      ,     pa.ArrowInvalid						)
               and "straddling" not in str(_SCREAMING_SNAKE_CASE						)
               or block_size > len(_SCREAMING_SNAKE_CASE						)
           ):
            raise
           else:
            # Increase the block size in case it was too small.
            # The block size will be reset for the next file.
            logger.debug(
                F"Batch of {len(_SCREAMING_SNAKE_CASE						)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."						)
            block_size *= 2
        except pa.ArrowInvalid as e:
         try:
          with open(
              _SCREAMING_SNAKE_CASE      ,     encoding=self.config.encoding      ,     errors=self.config.encoding_errors						) as f:
           UpperCAmelCase	:					str	=	json.load(_SCREAMING_SNAKE_CASE						)
         except json.JSONDecodeError:
          logger.error(F"Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE						)}: {e}"						)
          raise e
         # If possible, parse the file as a list of json objects and exit the loop
         if isinstance(_SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE						):  # list is the only sequence type supported in JSON
          try:
           UpperCAmelCase	:					Dict	=	set().union(*[row.keys() for row in dataset]						)
           UpperCAmelCase	:					Union[str, Any]	=	{col: [row.get(_SCREAMING_SNAKE_CASE						) for row in dataset] for col in keys}
           UpperCAmelCase	:					Dict	=	pa.Table.from_pydict(_SCREAMING_SNAKE_CASE						)
          except (pa.ArrowInvalid, AttributeError) as e:
           logger.error(F"Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE						)}: {e}"						)
           raise ValueError(F"Not able to read records in the JSON file at {file}."						) from None
          yield file_idx, self._cast_table(_SCREAMING_SNAKE_CASE						)
          break
         else:
          logger.error(F"Failed to read file '{file}' with error {type(_SCREAMING_SNAKE_CASE						)}: {e}"						)
          raise ValueError(
              F"Not able to read records in the JSON file at {file}. "
              F"You should probably indicate the field of the JSON file containing your records. "
              F"This JSON file contain the following fields: {str(list(dataset.keys()						)						)}. "
              F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "						) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
        yield (file_idx, batch_idx), self._cast_table(_SCREAMING_SNAKE_CASE						)
        batch_idx += 1
