


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE__     					=			datasets.utils.logging.get_logger(__name__)



@dataclass
class 						_UpperCAmelCase      (     datasets.BuilderConfig							):
		lowerCamelCase_	:			Optional[datasets.Features]       			=  None
		lowerCamelCase_	:			str       			=  "utf-8"
		lowerCamelCase_	:			Optional[str]       			=  None
		lowerCamelCase_	:			Optional[str]       			=  None
		lowerCamelCase_	:			bool       			=  True  # deprecated
		lowerCamelCase_	:			Optional[int]       			=  None  # deprecated
		lowerCamelCase_	:			int       			=  1_0 << 2_0  # 10MB
		lowerCamelCase_	:			Optional[bool]       			=  None




class 						_UpperCAmelCase      (     datasets.ArrowBasedBuilder							):
		lowerCamelCase_	:			Union[str, Any]       			=  JsonConfig






		def _snake_case		(						self				:     Any):
				if self.config.block_size is not None:
						logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead")
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						self.config.block_size
				if self.config.use_threads is not True:
						logger.warning(
						    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.")
				if self.config.newlines_in_values is not None:
						raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported")
				return datasets.DatasetInfo(features=self.config.features)






		def _snake_case		(						self				:     Optional[int]   ,  UpperCAmelCase				:     Union[str, Any]):
				if not self.config.data_files:
						raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
				SCREAMING_SNAKE_CASE_				:Tuple				     =						dl_manager.download_and_extract(self.config.data_files)
				if isinstance(UpperCAmelCase   ,  (str, list, tuple)):
						SCREAMING_SNAKE_CASE_				:Tuple				     =						data_files
						if isinstance(UpperCAmelCase   ,  UpperCAmelCase):
								SCREAMING_SNAKE_CASE_				:List[Any]				     =						[files]
						SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						[dl_manager.iter_files(UpperCAmelCase) for file in files]
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,  gen_kwargs={"files": files})]
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						[]
				for split_name, files in data_files.items():
						if isinstance(UpperCAmelCase   ,  UpperCAmelCase):
								SCREAMING_SNAKE_CASE_				:Any				     =						[files]
						SCREAMING_SNAKE_CASE_				:str				     =						[dl_manager.iter_files(UpperCAmelCase) for file in files]
						splits.append(datasets.SplitGenerator(name=UpperCAmelCase   ,  gen_kwargs={"files": files}))
				return splits






		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     pa.Table):
				if self.config.features is not None:
						# adding missing columns
						for column_name in set(self.config.features) - set(pa_table.column_names):
								SCREAMING_SNAKE_CASE_				:int				     =						self.config.features.arrow_schema.field(UpperCAmelCase).type
								SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						pa_table.append_column(UpperCAmelCase   ,  pa.array([None] * len(UpperCAmelCase)   ,  type=UpperCAmelCase))
						# more expensive cast to support nested structures with keys in a different order
						# allows str <-> int/float or str to Audio for example
						SCREAMING_SNAKE_CASE_				:Tuple				     =						table_cast(UpperCAmelCase   ,  self.config.features.arrow_schema)
				return pa_table







		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     int):
				for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase)):
						# If the file is one json object and if we need to look at the list of items in one specific field
						if self.config.field is not None:
								with open(UpperCAmelCase   ,  encoding=self.config.encoding   ,  errors=self.config.encoding_errors) as f:
										SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						json.load(UpperCAmelCase)

								# We keep only the field we are interested in
								SCREAMING_SNAKE_CASE_				:List[str]				     =						dataset[self.config.field]

								# We accept two format: a list of dicts or a dict of lists
								if isinstance(UpperCAmelCase   ,  (list, tuple)):
										SCREAMING_SNAKE_CASE_				:Dict				     =						set().union(*[row.keys() for row in dataset])
										SCREAMING_SNAKE_CASE_				:List[Any]				     =						{col: [row.get(UpperCAmelCase) for row in dataset] for col in keys}
								else:
										SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						dataset
								SCREAMING_SNAKE_CASE_				:Dict				     =						pa.Table.from_pydict(UpperCAmelCase)
								yield file_idx, self._cast_table(UpperCAmelCase)

						# If the file has one json object per line
						else:
								with open(UpperCAmelCase   ,  "rb") as f:
										SCREAMING_SNAKE_CASE_				:Dict				     =						0
										# Use block_size equal to the chunk size divided by 32 to leverage multithreading
										# Set a default minimum value of 16kB if the chunk size is really small
										SCREAMING_SNAKE_CASE_				:Tuple				     =						max(self.config.chunksize // 32   ,  16 << 10)
										SCREAMING_SNAKE_CASE_				:List[str]				     =						(
										    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
										)
										while True:
												SCREAMING_SNAKE_CASE_				:List[str]				     =						f.read(self.config.chunksize)
												if not batch:
														break
												# Finish current line
												try:
														batch += f.readline()
												except (AttributeError, io.UnsupportedOperation):
														batch += readline(UpperCAmelCase)
												# PyArrow only accepts utf-8 encoded bytes
												if self.config.encoding != "utf-8":
														SCREAMING_SNAKE_CASE_				:Any				     =						batch.decode(self.config.encoding   ,  errors=UpperCAmelCase).encode("utf-8")
												try:
														while True:
																try:
																		SCREAMING_SNAKE_CASE_				:Dict				     =						paj.read_json(
																		    io.BytesIO(UpperCAmelCase)   ,  read_options=paj.ReadOptions(block_size=UpperCAmelCase))
																		break
																except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																		if (
																		    isinstance(UpperCAmelCase   ,  pa.ArrowInvalid)
																		    and "straddling" not in str(UpperCAmelCase)
																		    or block_size > len(UpperCAmelCase)
																		):
																				raise
																		else:
																				# Increase the block size in case it was too small.
																				# The block size will be reset for the next file.
																				logger.debug(
																				    F"Batch of {len(UpperCAmelCase)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.")
																				block_size *= 2
												except pa.ArrowInvalid as e:
														try:
																with open(
																    UpperCAmelCase   ,  encoding=self.config.encoding   ,  errors=self.config.encoding_errors) as f:
																		SCREAMING_SNAKE_CASE_				:int				     =						json.load(UpperCAmelCase)
														except json.JSONDecodeError:
																logger.error(F"Failed to read file '{file}' with error {type(UpperCAmelCase)}: {e}")
																raise e
														# If possible, parse the file as a list of json objects and exit the loop
														if isinstance(UpperCAmelCase   ,  UpperCAmelCase):  # list is the only sequence type supported in JSON
																try:
																		SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						set().union(*[row.keys() for row in dataset])
																		SCREAMING_SNAKE_CASE_				:Tuple				     =						{col: [row.get(UpperCAmelCase) for row in dataset] for col in keys}
																		SCREAMING_SNAKE_CASE_				:Dict				     =						pa.Table.from_pydict(UpperCAmelCase)
																except (pa.ArrowInvalid, AttributeError) as e:
																		logger.error(F"Failed to read file '{file}' with error {type(UpperCAmelCase)}: {e}")
																		raise ValueError(F"Not able to read records in the JSON file at {file}.") from None
																yield file_idx, self._cast_table(UpperCAmelCase)
																break
														else:
																logger.error(F"Failed to read file '{file}' with error {type(UpperCAmelCase)}: {e}")
																raise ValueError(
																    F"Not able to read records in the JSON file at {file}. "
																    F"You should probably indicate the field of the JSON file containing your records. "
																    F"This JSON file contain the following fields: {str(list(dataset.keys()))}. "
																    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. ") from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
												yield (file_idx, batch_idx), self._cast_table(UpperCAmelCase)
												batch_idx += 1










from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
		import PIL


SCREAMING_SNAKE_CASE__     					=			logging.get_logger(__name__)


def     lowercase	(    a     , a ):



		'''simple docstring'''




		SCREAMING_SNAKE_CASE_				:Dict				     =						b.T
		SCREAMING_SNAKE_CASE_				:str				     =						np.sum(np.square(a )     , axis=1 )
		SCREAMING_SNAKE_CASE_				:List[Any]				     =						np.sum(np.square(a )     , axis=0 )
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						np.matmul(a     , a )
		SCREAMING_SNAKE_CASE_				:Any				     =						aa[:, None] - 2 * ab + ba[None, :]
		return d


def     lowercase	(    a     , a ):



		'''simple docstring'''




		SCREAMING_SNAKE_CASE_				:str				     =						x.reshape(-1     , 3 )
		SCREAMING_SNAKE_CASE_				:str				     =						squared_euclidean_distance(a     , a )
		return np.argmin(a     , axis=1 )



class 						_UpperCAmelCase      (     lowercase							):
		lowerCamelCase_	:			List[str]       			=  ["""pixel_values"""]






		def __init__(						self				:     Dict   ,  UpperCAmelCase				:     Optional[Union[List[List[int]], np.ndarray]] = None   ,  UpperCAmelCase				:     bool = True   ,  UpperCAmelCase				:     Dict[str, int] = None   ,  UpperCAmelCase				:     PILImageResampling = PILImageResampling.BILINEAR   ,  UpperCAmelCase				:     bool = True   ,  UpperCAmelCase				:     bool = True   ,  **UpperCAmelCase				:     Union[str, Any]   ,  ):
				super().__init__(**UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						size if size is not None else {"height": 2_56, "width": 2_56}
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						get_size_dict(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:int				     =						np.array(UpperCAmelCase) if clusters is not None else None
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						do_resize
				SCREAMING_SNAKE_CASE_				:Dict				     =						size
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						resample
				SCREAMING_SNAKE_CASE_				:Dict				     =						do_normalize
				SCREAMING_SNAKE_CASE_				:Tuple				     =						do_color_quantize






		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     np.ndarray   ,  UpperCAmelCase				:     Dict[str, int]   ,  UpperCAmelCase				:     PILImageResampling = PILImageResampling.BILINEAR   ,  UpperCAmelCase				:     Optional[Union[str, ChannelDimension]] = None   ,  **UpperCAmelCase				:     List[Any]   ,  ):
				SCREAMING_SNAKE_CASE_				:str				     =						get_size_dict(UpperCAmelCase)
				if "height" not in size or "width" not in size:
						raise ValueError(F"Size dictionary must contain both height and width keys. Got {size.keys()}")
				return resize(
				    UpperCAmelCase   ,  size=(size["height"], size["width"])   ,  resample=UpperCAmelCase   ,  data_format=UpperCAmelCase   ,  **UpperCAmelCase)






		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     np.ndarray   ,  UpperCAmelCase				:     Optional[Union[str, ChannelDimension]] = None   ,  ):
				SCREAMING_SNAKE_CASE_				:int				     =						rescale(image=UpperCAmelCase   ,  scale=1 / 127.5   ,  data_format=UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:List[str]				     =						image - 1
				return image







		def _snake_case		(						self				:     List[str]   ,  UpperCAmelCase				:     ImageInput   ,  UpperCAmelCase				:     bool = None   ,  UpperCAmelCase				:     Dict[str, int] = None   ,  UpperCAmelCase				:     PILImageResampling = None   ,  UpperCAmelCase				:     bool = None   ,  UpperCAmelCase				:     Optional[bool] = None   ,  UpperCAmelCase				:     Optional[Union[List[List[int]], np.ndarray]] = None   ,  UpperCAmelCase				:     Optional[Union[str, TensorType]] = None   ,  UpperCAmelCase				:     Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST   ,  **UpperCAmelCase				:     Optional[Any]   ,  ):
				SCREAMING_SNAKE_CASE_				:Any				     =						do_resize if do_resize is not None else self.do_resize
				SCREAMING_SNAKE_CASE_				:Dict				     =						size if size is not None else self.size
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						get_size_dict(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						resample if resample is not None else self.resample
				SCREAMING_SNAKE_CASE_				:Dict				     =						do_normalize if do_normalize is not None else self.do_normalize
				SCREAMING_SNAKE_CASE_				:List[str]				     =						do_color_quantize if do_color_quantize is not None else self.do_color_quantize
				SCREAMING_SNAKE_CASE_				:Tuple				     =						clusters if clusters is not None else self.clusters
				SCREAMING_SNAKE_CASE_				:Tuple				     =						np.array(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Dict				     =						make_list_of_images(UpperCAmelCase)

				if not valid_images(UpperCAmelCase):
						raise ValueError(
						    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
						    "torch.Tensor, tf.Tensor or jax.ndarray.")

				if do_resize and size is None or resample is None:
						raise ValueError("Size and resample must be specified if do_resize is True.")

				if do_color_quantize and clusters is None:
						raise ValueError("Clusters must be specified if do_color_quantize is True.")

				# All transformations expect numpy arrays.
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						[to_numpy_array(UpperCAmelCase) for image in images]

				if do_resize:
						SCREAMING_SNAKE_CASE_				:Tuple				     =						[self.resize(image=UpperCAmelCase   ,  size=UpperCAmelCase   ,  resample=UpperCAmelCase) for image in images]

				if do_normalize:
						SCREAMING_SNAKE_CASE_				:int				     =						[self.normalize(image=UpperCAmelCase) for image in images]

				if do_color_quantize:
						SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						[to_channel_dimension_format(UpperCAmelCase   ,  ChannelDimension.LAST) for image in images]
						# color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
						SCREAMING_SNAKE_CASE_				:Optional[int]				     =						np.array(UpperCAmelCase)
						SCREAMING_SNAKE_CASE_				:Tuple				     =						color_quantize(UpperCAmelCase   ,  UpperCAmelCase).reshape(images.shape[:-1])

						# flatten to (batch_size, height*width)
						SCREAMING_SNAKE_CASE_				:Any				     =						images.shape[0]
						SCREAMING_SNAKE_CASE_				:Dict				     =						images.reshape(UpperCAmelCase   ,  -1)

						# We need to convert back to a list of images to keep consistent behaviour across processors.
						SCREAMING_SNAKE_CASE_				:Any				     =						list(UpperCAmelCase)
				else:
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						[to_channel_dimension_format(UpperCAmelCase   ,  UpperCAmelCase) for image in images]

				SCREAMING_SNAKE_CASE_				:str				     =						{"input_ids": images}
				return BatchFeature(data=UpperCAmelCase   ,  tensor_type=UpperCAmelCase)






