
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase__							:							int													=	datasets.utils.logging.get_logger(__name__)






@dataclass
class 				SCREAMING_SNAKE_CASE__			(							datasets.BuilderConfig			):

							"""simple docstring"""






							_snake_case   						=     None
							_snake_case   						=     'utf-8'
							_snake_case   						=     None
							_snake_case   						=     None
							_snake_case   						=     True  # deprecated
							_snake_case   						=     None  # deprecated
							_snake_case   						=     10 << 20  # 10MB
							_snake_case   						=     None






class 				SCREAMING_SNAKE_CASE__			(							datasets.ArrowBasedBuilder			):

							"""simple docstring"""






							_snake_case   						=     JsonConfig



							def 			A__						(     self	)->      Optional[int]:





													'''simple docstring'''






													if self.config.block_size is not None:
																			logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''	)
																			__UpperCamelCase        =	self.config.block_size
													if self.config.use_threads is not True:
																			logger.warning(
																			    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''	)
													if self.config.newlines_in_values is not None:
																			raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''	)
													return datasets.DatasetInfo(features=self.config.features	)



							def 			A__						(     self   ,		SCREAMING_SNAKE_CASE_	)->      str:





													'''simple docstring'''






													if not self.config.data_files:
																			raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"	)
													__UpperCamelCase        =	dl_manager.download_and_extract(self.config.data_files	)
													if isinstance(__SCREAMING_SNAKE_CASE   ,		(str, list, tuple)	):
																			__UpperCamelCase        =	data_files
																			if isinstance(__SCREAMING_SNAKE_CASE   ,		__SCREAMING_SNAKE_CASE	):
																									__UpperCamelCase        =	[files]
																			__UpperCamelCase        =	[dl_manager.iter_files(__SCREAMING_SNAKE_CASE	) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,		gen_kwargs={'''files''': files}	)]
													__UpperCamelCase        =	[]
													for split_name, files in data_files.items():
																			if isinstance(__SCREAMING_SNAKE_CASE   ,		__SCREAMING_SNAKE_CASE	):
																									__UpperCamelCase        =	[files]
																			__UpperCamelCase        =	[dl_manager.iter_files(__SCREAMING_SNAKE_CASE	) for file in files]
																			splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE   ,		gen_kwargs={'''files''': files}	)	)
													return splits



							def 			A__						(     self   ,		SCREAMING_SNAKE_CASE_	)->      List[str]:





													'''simple docstring'''






													if self.config.features is not None:
																			# adding missing columns
																			for column_name in set(self.config.features	) - set(pa_table.column_names	):
																									__UpperCamelCase        =	self.config.features.arrow_schema.field(__SCREAMING_SNAKE_CASE	).type
																									__UpperCamelCase        =	pa_table.append_column(__SCREAMING_SNAKE_CASE   ,		pa.array([None] * len(__SCREAMING_SNAKE_CASE	)   ,		type=__SCREAMING_SNAKE_CASE	)	)
																			# more expensive cast to support nested structures with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			__UpperCamelCase        =	table_cast(__SCREAMING_SNAKE_CASE   ,		self.config.features.arrow_schema	)
													return pa_table



							def 			A__						(     self   ,		SCREAMING_SNAKE_CASE_	)->      Optional[int]:





													'''simple docstring'''






													for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE	)	):
																			# If the file is one json object and if we need to look at the list of items in one specific field
																			if self.config.field is not None:
																									with open(__SCREAMING_SNAKE_CASE   ,		encoding=self.config.encoding   ,		errors=self.config.encoding_errors	) as f:
																															__UpperCamelCase        =	json.load(__SCREAMING_SNAKE_CASE	)

																									# We keep only the field we are interested in
																									__UpperCamelCase        =	dataset[self.config.field]

																									# We accept two format: a list of dicts or a dict of lists
																									if isinstance(__SCREAMING_SNAKE_CASE   ,		(list, tuple)	):
																															__UpperCamelCase        =	set().union(*[row.keys() for row in dataset]	)
																															__UpperCamelCase        =	{col: [row.get(__SCREAMING_SNAKE_CASE	) for row in dataset] for col in keys}
																									else:
																															__UpperCamelCase        =	dataset
																									__UpperCamelCase        =	pa.Table.from_pydict(__SCREAMING_SNAKE_CASE	)
																									yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE	)

																			# If the file has one json object per line
																			else:
																									with open(__SCREAMING_SNAKE_CASE   ,		'''rb'''	) as f:
																															__UpperCamelCase        =	0
																															# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																															# Set a default minimum value of 16kB if the chunk size is really small
																															__UpperCamelCase        =	max(self.config.chunksize // 32   ,		16 << 10	)
																															__UpperCamelCase        =	(
																															    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																															)
																															while True:
																																					__UpperCamelCase        =	f.read(self.config.chunksize	)
																																					if not batch:
																																											break
																																					# Finish current line
																																					try:
																																											batch += f.readline()
																																					except (AttributeError, io.UnsupportedOperation):
																																											batch += readline(__SCREAMING_SNAKE_CASE	)
																																					# PyArrow only accepts utf-8 encoded bytes
																																					if self.config.encoding != "utf-8":
																																											__UpperCamelCase        =	batch.decode(self.config.encoding   ,		errors=__SCREAMING_SNAKE_CASE	).encode('''utf-8'''	)
																																					try:
																																											while True:
																																																	try:
																																																							__UpperCamelCase        =	paj.read_json(
																																																							    io.BytesIO(__SCREAMING_SNAKE_CASE	)   ,		read_options=paj.ReadOptions(block_size=__SCREAMING_SNAKE_CASE	)	)
																																																							break
																																																	except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																							if (
																																																							    isinstance(__SCREAMING_SNAKE_CASE   ,		pa.ArrowInvalid	)
																																																							    and "straddling" not in str(__SCREAMING_SNAKE_CASE	)
																																																							    or block_size > len(__SCREAMING_SNAKE_CASE	)
																																																							):
																																																													raise
																																																							else:
																																																													# Increase the block size in case it was too small.
																																																													# The block size will be reset for the next file.
																																																													logger.debug(
																																																													    F"Batch of {len(__SCREAMING_SNAKE_CASE	)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."	)
																																																													block_size *= 2
																																					except pa.ArrowInvalid as e:
																																											try:
																																																	with open(
																																																	    __SCREAMING_SNAKE_CASE   ,		encoding=self.config.encoding   ,		errors=self.config.encoding_errors	) as f:
																																																							__UpperCamelCase        =	json.load(__SCREAMING_SNAKE_CASE	)
																																											except json.JSONDecodeError:
																																																	logger.error(F"Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE	)}: {e}"	)
																																																	raise e
																																											# If possible, parse the file as a list of json objects and exit the loop
																																											if isinstance(__SCREAMING_SNAKE_CASE   ,		__SCREAMING_SNAKE_CASE	):  # list is the only sequence type supported in JSON
																																																	try:
																																																							__UpperCamelCase        =	set().union(*[row.keys() for row in dataset]	)
																																																							__UpperCamelCase        =	{col: [row.get(__SCREAMING_SNAKE_CASE	) for row in dataset] for col in keys}
																																																							__UpperCamelCase        =	pa.Table.from_pydict(__SCREAMING_SNAKE_CASE	)
																																																	except (pa.ArrowInvalid, AttributeError) as e:
																																																							logger.error(F"Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE	)}: {e}"	)
																																																							raise ValueError(F"Not able to read records in the JSON file at {file}."	) from None
																																																	yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE	)
																																																	break
																																											else:
																																																	logger.error(F"Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE	)}: {e}"	)
																																																	raise ValueError(
																																																	    F"Not able to read records in the JSON file at {file}. "
																																																	    F"You should probably indicate the field of the JSON file containing your records. "
																																																	    F"This JSON file contain the following fields: {str(list(dataset.keys()	)	)}. "
																																																	    F"Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. "	) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																					yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE	)
																																					batch_idx += 1






'''simple docstring'''







import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
							import torch_xla.debug.metrics as met






class  lowerCAmelCase__   (  lowerCamelCase_     ):






					def __init__(   self				,							*__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE=None				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											super().__init__(*__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Tuple													=       eval_examples
											lowercase_	:     Tuple													=       post_process_function






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE = "eval"				,							**__SCREAMING_SNAKE_CASE				,							):


											"""simple docstring"""





											lowercase_	:     Optional[int]													=       gen_kwargs.copy()
											lowercase_	:     List[str]													=       (
											    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''		) is not None else self.args.generation_max_length
											)
											lowercase_	:     str													=       (
											    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''		) is not None else self.args.generation_num_beams
											)
											lowercase_	:     Dict													=       gen_kwargs

											lowercase_	:     List[Any]													=       self.eval_dataset if eval_dataset is None else eval_dataset
											lowercase_	:     List[str]													=       self.get_eval_dataloader(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Optional[Any]													=       self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											lowercase_	:     Union[str, Any]													=       self.compute_metrics
											lowercase_	:     Optional[int]													=       None
											lowercase_	:     Tuple													=       time.time()
											lowercase_	:     Tuple													=       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																	lowercase_	:     str													=       eval_loop(
																	    __SCREAMING_SNAKE_CASE				,							description='''Evaluation'''				,							prediction_loss_only=True if compute_metrics is None else None				,							ignore_keys=__SCREAMING_SNAKE_CASE				,							metric_key_prefix=__SCREAMING_SNAKE_CASE				,							)
											finally:
																	lowercase_	:     Any													=       compute_metrics
											lowercase_	:     Any													=       self.args.eval_batch_size * self.args.world_size
											if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
																	start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
											output.metrics.update(
											    speed_metrics(
											        __SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							num_samples=output.num_samples				,							num_steps=math.ceil(output.num_samples / total_batch_size		)				,							)		)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																	# Only the main node write the results by default
																	lowercase_	:     Optional[Any]													=       self.post_process_function(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     Optional[int]													=       self.compute_metrics(__SCREAMING_SNAKE_CASE		)

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()		):
																							if not key.startswith(F'''{metric_key_prefix}_'''		):
																													lowercase_	:     List[Any]													=       metrics.pop(__SCREAMING_SNAKE_CASE		)

																	metrics.update(output.metrics		)
											else:
																	lowercase_	:     List[Any]													=       output.metrics

											if self.args.should_log:
																	# Only the main node log the results by default
																	self.log(__SCREAMING_SNAKE_CASE		)

											if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report()		)

											lowercase_	:     List[Any]													=       self.callback_handler.on_evaluate(self.args				,							self.state				,							self.control				,							__SCREAMING_SNAKE_CASE		)
											return metrics






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE = "test"				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Union[str, Any]													=       gen_kwargs.copy()

											lowercase_	:     Tuple													=       self.get_test_dataloader(__SCREAMING_SNAKE_CASE		)

											# Temporarily disable metric computation, we will do it in the loop here.
											lowercase_	:     Optional[Any]													=       self.compute_metrics
											lowercase_	:     Optional[int]													=       None
											lowercase_	:     List[Any]													=       time.time()
											lowercase_	:     int													=       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																	lowercase_	:     Tuple													=       eval_loop(
																	    __SCREAMING_SNAKE_CASE				,							description='''Prediction'''				,							prediction_loss_only=True if compute_metrics is None else None				,							ignore_keys=__SCREAMING_SNAKE_CASE				,							metric_key_prefix=__SCREAMING_SNAKE_CASE				,							)
											finally:
																	lowercase_	:     Any													=       compute_metrics

											lowercase_	:     Tuple													=       self.args.eval_batch_size * self.args.world_size
											if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
																	start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
											output.metrics.update(
											    speed_metrics(
											        __SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							num_samples=output.num_samples				,							num_steps=math.ceil(output.num_samples / total_batch_size		)				,							)		)
											if self.post_process_function is None or self.compute_metrics is None:
																	return output

											lowercase_	:     Any													=       self.post_process_function(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							'''predict'''		)
											lowercase_	:     str													=       self.compute_metrics(__SCREAMING_SNAKE_CASE		)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()		):
																	if not key.startswith(F'''{metric_key_prefix}_'''		):
																							lowercase_	:     Optional[int]													=       metrics.pop(__SCREAMING_SNAKE_CASE		)
											metrics.update(output.metrics		)
											return PredictionOutput(predictions=predictions.predictions				,							label_ids=predictions.label_ids				,							metrics=__SCREAMING_SNAKE_CASE		)



