




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class 			a__    (					snake_case__      ):






						def __init__(   self		,			_A		,			_A = None		,			_A = None		,			_A = None		,			_A = False		,			_A = False		,			_A = None		,			_A = None		,			**_A		,			):




											"""simple docstring"""



											super().__init__(
											    _A		,			split=_A		,			features=_A		,			cache_dir=_A		,			keep_in_memory=_A		,			streaming=_A		,			num_proc=_A		,			**_A		,			)
											__lowerCAmelCase				   =       field
											__lowerCAmelCase				   =       path_or_paths if isinstance(_A		,			_A			) else {self.split: path_or_paths}
											__lowerCAmelCase				   =       Json(
											    cache_dir=_A		,			data_files=_A		,			features=_A		,			field=_A		,			**_A		,			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											if self.streaming:
																__lowerCAmelCase				   =       self.builder.as_streaming_dataset(split=self.split			)
											# Build regular (map-style) dataset
											else:
																__lowerCAmelCase				   =       None
																__lowerCAmelCase				   =       None
																__lowerCAmelCase				   =       None
																__lowerCAmelCase				   =       None

																self.builder.download_and_prepare(
																    download_config=_A		,			download_mode=_A		,			verification_mode=_A		,			base_path=_A		,			num_proc=self.num_proc		,			)
																__lowerCAmelCase				   =       self.builder.as_dataset(
																    split=self.split		,			verification_mode=_A		,			in_memory=self.keep_in_memory			)
											return dataset






class 			a__    :






						def __init__(   self		,			_A		,			_A		,			_A = None		,			_A = None		,			**_A		,			):




											"""simple docstring"""



											if num_proc is not None and num_proc <= 0:
																raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""			)

											__lowerCAmelCase				   =       dataset
											__lowerCAmelCase				   =       path_or_buf
											__lowerCAmelCase				   =       batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											__lowerCAmelCase				   =       num_proc
											__lowerCAmelCase				   =       "utf-8"
											__lowerCAmelCase				   =       to_json_kwargs






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       self.to_json_kwargs.pop("path_or_buf"		,			_A			)
											__lowerCAmelCase				   =       self.to_json_kwargs.pop("orient"		,			"records"			)
											__lowerCAmelCase				   =       self.to_json_kwargs.pop("lines"		,			True if orient == "records" else False			)
											__lowerCAmelCase				   =       self.to_json_kwargs.pop("index"		,			False if orient in ["split", "table"] else True			)
											__lowerCAmelCase				   =       self.to_json_kwargs.pop("compression"		,			_A			)

											if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																raise NotImplementedError(f"""`datasets` currently does not support {compression} compression"""			)

											if isinstance(self.path_or_buf		,			(str, bytes, os.PathLike)			):
																with fsspec.open(self.path_or_buf		,			"wb"		,			compression=_A			) as buffer:
																					__lowerCAmelCase				   =       self._write(file_obj=_A		,			orient=_A		,			lines=_A		,			index=_A		,			**self.to_json_kwargs			)
											else:
																if compression:
																					raise NotImplementedError(
																					    f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
																					    " was passed. Please provide a local path instead."			)
																__lowerCAmelCase				   =       self._write(
																    file_obj=self.path_or_buf		,			orient=_A		,			lines=_A		,			index=_A		,			**self.to_json_kwargs			)
											return written






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase				   =       args

											__lowerCAmelCase				   =       query_table(
											    table=self.dataset.data		,			key=slice(_A		,			offset + self.batch_size			)		,			indices=self.dataset._indices		,			)
											__lowerCAmelCase				   =       batch.to_pandas().to_json(
											    path_or_buf=_A		,			orient=_A		,			lines=_A		,			index=_A		,			**_A			)
											if not json_str.endswith("\n"			):
																json_str += "\n"
											return json_str.encode(self.encoding			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A		,			_A		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       0

											if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0		,			len(self.dataset			)		,			self.batch_size			)		,			unit="ba"		,			disable=not logging.is_progress_bar_enabled()		,			desc="Creating json from Arrow format"		,			):
																					__lowerCAmelCase				   =       self._batch_json((offset, orient, lines, index, to_json_kwargs)			)
																					written += file_obj.write(_A			)
											else:
																__lowerCAmelCase   ,							__lowerCAmelCase				   =       len(self.dataset			), self.batch_size
																with multiprocessing.Pool(self.num_proc			) as pool:
																					for json_str in logging.tqdm(
																					    pool.imap(
																					        self._batch_json		,			[(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,			_A		,			_A			)]		,			)		,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,			unit="ba"		,			disable=not logging.is_progress_bar_enabled()		,			desc="Creating json from Arrow format"		,			):
																										written += file_obj.write(_A			)

											return written










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm
					import torch_xla.debug.metrics as met
class 			a__    (					snake_case__      ):






						def __init__(   self		,			*_A		,			_A=None		,			_A=None		,			**_A			):




											"""simple docstring"""



											super().__init__(*_A		,			**_A			)
											__lowerCAmelCase				   =       eval_examples
											__lowerCAmelCase				   =       post_process_function






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A = None		,			_A=None		,			_A = None		,			_A = "eval"		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()
											__lowerCAmelCase				   =       (
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"			) is not None else self.args.generation_max_length
											)
											__lowerCAmelCase				   =       (
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"			) is not None else self.args.generation_num_beams
											)
											__lowerCAmelCase				   =       gen_kwargs

											__lowerCAmelCase				   =       self.eval_dataset if eval_dataset is None else eval_dataset
											__lowerCAmelCase				   =       self.get_eval_dataloader(_A			)
											__lowerCAmelCase				   =       self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Evaluation"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics
											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A			)
																__lowerCAmelCase				   =       self.compute_metrics(_A			)

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()			):
																					if not key.startswith(f"""{metric_key_prefix}_"""			):
																										__lowerCAmelCase				   =       metrics.pop(_A			)

																metrics.update(output.metrics			)
											else:
																__lowerCAmelCase				   =       output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(_A			)

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()			)

											__lowerCAmelCase				   =       self.callback_handler.on_evaluate(self.args		,			self.state		,			self.control		,			_A			)
											return metrics






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A=None		,			_A = "test"		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()

											__lowerCAmelCase				   =       self.get_test_dataloader(_A			)

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Prediction"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics

											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A		,			"predict"			)
											__lowerCAmelCase				   =       self.compute_metrics(_A			)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()			):
																if not key.startswith(f"""{metric_key_prefix}_"""			):
																					__lowerCAmelCase				   =       metrics.pop(_A			)
											metrics.update(output.metrics			)
											return PredictionOutput(predictions=predictions.predictions		,			label_ids=predictions.label_ids		,			metrics=_A			)




