





"""simple docstring"""


import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


__magic_name__       :					str						=			datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
		import pyspark






@dataclass
class 							lowercase__			(   datasets.BuilderConfig	):

			"""simple docstring"""
			__lowerCAmelCase     :				Optional[datasets.Features]										=  None


def      UpperCamelCase							(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE							,     ):
						import pyspark

						def generate_fn():
												UpperCamelCase				:		List[str]				   =			df.select("""*"""							,     pyspark.sql.functions.spark_partition_id().alias("""part_id"""     )     )
												for partition_id in partition_order:
																		UpperCamelCase				:		List[str]				   =			df_with_partition_id.select("""*"""     ).where(f"""part_id = {partition_id}"""     ).drop("""part_id"""     )
																		UpperCamelCase				:		Optional[Any]				   =			partition_df.collect()
																		UpperCamelCase				:		Union[str, Any]				   =			0
																		for row in rows:
																								yield f"""{partition_id}_{row_id}""", row.asDict()
																								row_id += 1

						return generate_fn






class 							lowercase__			(   _BaseExamplesIterable	):

			"""simple docstring"""







			def __init__(      self						,   _A						,   _A=None						,   ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			df
									UpperCamelCase				:		Tuple				   =			partition_order or range(self.df.rdd.getNumPartitions()    )
									UpperCamelCase				:		Union[str, Any]				   =			_generate_iterable_examples(self.df						,   self.partition_order    )







			def __iter__(      self    ):






									'''simple docstring'''







									yield from self.generate_examples_fn()







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			list(range(self.df.rdd.getNumPartitions()    )    )
									generator.shuffle(_A    )
									return SparkExamplesIterable(self.df						,   partition_order=_A    )







			def   _a      (      self						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[int]				   =			self.split_shard_indices_by_worker(_A						,   _A    )
									return SparkExamplesIterable(self.df						,   partition_order=_A    )







			@property
			def   _a      (      self    ):






									'''simple docstring'''







									return len(self.partition_order    )






class 							lowercase__			(   datasets.DatasetBuilder	):

			"""simple docstring"""
			__lowerCAmelCase     :				Dict										=  SparkConfig







			def __init__(      self						,   _A						,   _A = None						,   _A = None						,   **_A						,   ):






									'''simple docstring'''







									import pyspark

									UpperCamelCase				:		Optional[int]				   =			pyspark.sql.SparkSession.builder.getOrCreate()
									UpperCamelCase				:		List[str]				   =			df
									UpperCamelCase				:		Optional[Any]				   =			working_dir

									super().__init__(
									    cache_dir=_A						,   config_name=str(self.df.semanticHash()    )						,   **_A						,   )







			def   _a      (      self    ):






									'''simple docstring'''







									def create_cache_and_write_probe(_A    ):
															# makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
															# already exist.
															os.makedirs(self._cache_dir						,   exist_ok=_A    )
															UpperCamelCase				:		List[Any]				   =			os.path.join(self._cache_dir						,   """fs_test""" + uuid.uuida().hex    )
															# Opening the file in append mode will create a new file unless it already exists, in which case it will not
															# change the file contents.
															open(_A						,   """a"""    )
															return [probe_file]

									if self._spark.conf.get("""spark.master"""						,   """"""    ).startswith("""local"""    ):
															return

									# If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
									# accessible to the driver.
									# TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
									if self._cache_dir:
															UpperCamelCase				:		Tuple				   =			(
															    self._spark.sparkContext.parallelize(range(1    )						,   1    ).mapPartitions(_A    ).collect()
															)
															if os.path.isfile(probe[0]    ):
																					return

									raise ValueError(
									    """When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"""    )







			def   _a      (      self    ):






									'''simple docstring'''







									return datasets.DatasetInfo(features=self.config.features    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									return [datasets.SplitGenerator(name=datasets.Split.TRAIN    )]







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									import pyspark

									def get_arrow_batch_size(_A    ):
															for batch in it:
																					yield pa.RecordBatch.from_pydict({"""batch_bytes""": [batch.nbytes]}    )

									UpperCamelCase				:		Tuple				   =			self.df.count()
									UpperCamelCase				:		Union[str, Any]				   =			df_num_rows if df_num_rows <= 1_0_0 else 1_0_0
									# Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
									UpperCamelCase				:		List[Any]				   =			(
									    self.df.limit(_A    )
									    .repartition(1    )
									    .mapInArrow(_A						,   """batch_bytes: long"""    )
									    .agg(pyspark.sql.functions.sum("""batch_bytes"""    ).alias("""sample_bytes"""    )    )
									    .collect()[0]
									    .sample_bytes
									    / sample_num_rows
									)
									UpperCamelCase				:		str				   =			approx_bytes_per_row * df_num_rows
									if approx_total_size > max_shard_size:
															# Make sure there is at least one row per partition.
															UpperCamelCase				:		int				   =			min(_A						,   int(approx_total_size / max_shard_size    )    )
															UpperCamelCase				:		Optional[Any]				   =			self.df.repartition(_A    )







			def   _a      (      self						,   _A						,   _A						,   _A						,   ):






									'''simple docstring'''







									import pyspark

									UpperCamelCase				:		List[Any]				   =			ParquetWriter if file_format == """parquet""" else ArrowWriter
									UpperCamelCase				:		Dict				   =			os.path.join(self._working_dir						,   os.path.basename(_A    )    ) if self._working_dir else fpath
									UpperCamelCase				:		List[Any]				   =			file_format == """parquet"""

									# Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
									# pickling the SparkContext.
									UpperCamelCase				:		Optional[int]				   =			self.config.features
									UpperCamelCase				:		Optional[Any]				   =			self._writer_batch_size
									UpperCamelCase				:		Union[str, Any]				   =			self._fs.storage_options

									def write_arrow(_A    ):
															# Within the same SparkContext, no two task attempts will share the same attempt ID.
															UpperCamelCase				:		Optional[int]				   =			pyspark.TaskContext().taskAttemptId()
															UpperCamelCase				:		str				   =			next(_A						,   _A    )
															if first_batch is None:
																					# Some partitions might not receive any data.
																					return pa.RecordBatch.from_arrays(
																					    [[task_id], [0], [0]]						,   names=["""task_id""", """num_examples""", """num_bytes"""]						,   )
															UpperCamelCase				:		str				   =			0
															UpperCamelCase				:		Union[str, Any]				   =			writer_class(
															    features=_A						,   path=working_fpath.replace("""SSSSS"""						,   f"""{shard_id:05d}"""    ).replace("""TTTTT"""						,   f"""{task_id:05d}"""    )						,   writer_batch_size=_A						,   storage_options=_A						,   embed_local_files=_A						,   )
															UpperCamelCase				:		List[str]				   =			pa.Table.from_batches([first_batch]    )
															writer.write_table(_A    )
															for batch in it:
																					if max_shard_size is not None and writer._num_bytes >= max_shard_size:
																											UpperCamelCase   ,						UpperCamelCase				:		Any				   =			writer.finalize()
																											writer.close()
																											yield pa.RecordBatch.from_arrays(
																											    [[task_id], [num_examples], [num_bytes]]						,   names=["""task_id""", """num_examples""", """num_bytes"""]						,   )
																											shard_id += 1
																											UpperCamelCase				:		Any				   =			writer_class(
																											    features=writer._features						,   path=working_fpath.replace("""SSSSS"""						,   f"""{shard_id:05d}"""    ).replace("""TTTTT"""						,   f"""{task_id:05d}"""    )						,   writer_batch_size=_A						,   storage_options=_A						,   embed_local_files=_A						,   )
																					UpperCamelCase				:		Dict				   =			pa.Table.from_batches([batch]    )
																					writer.write_table(_A    )

															if writer._num_bytes > 0:
																					UpperCamelCase   ,						UpperCamelCase				:		Any				   =			writer.finalize()
																					writer.close()
																					yield pa.RecordBatch.from_arrays(
																					    [[task_id], [num_examples], [num_bytes]]						,   names=["""task_id""", """num_examples""", """num_bytes"""]						,   )

															if working_fpath != fpath:
																					for file in os.listdir(os.path.dirname(_A    )    ):
																											UpperCamelCase				:		Optional[int]				   =			os.path.join(os.path.dirname(_A    )						,   os.path.basename(_A    )    )
																											shutil.move(_A						,   _A    )

									UpperCamelCase				:		int				   =			(
									    self.df.mapInArrow(_A						,   """task_id: long, num_examples: long, num_bytes: long"""    )
									    .groupBy("""task_id"""    )
									    .agg(
									        pyspark.sql.functions.sum("""num_examples"""    ).alias("""total_num_examples"""    )						,   pyspark.sql.functions.sum("""num_bytes"""    ).alias("""total_num_bytes"""    )						,   pyspark.sql.functions.count("""num_bytes"""    ).alias("""num_shards"""    )						,   pyspark.sql.functions.collect_list("""num_examples"""    ).alias("""shard_lengths"""    )						,   )
									    .collect()
									)
									for row in stats:
															yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)







			def   _a      (      self						,   _A						,   _A = "arrow"						,   _A = None						,   _A = None						,   **_A						,   ):






									'''simple docstring'''







									self._validate_cache_dir()

									UpperCamelCase				:		Dict				   =			convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE    )
									self._repartition_df_if_needed(_A    )
									UpperCamelCase				:		Optional[int]				   =			not is_remote_filesystem(self._fs    )
									UpperCamelCase				:		Dict				   =			os.path.join if is_local else posixpath.join

									UpperCamelCase				:		Optional[int]				   =			"""-TTTTT-SSSSS-of-NNNNN"""
									UpperCamelCase				:		Dict				   =			f"""{self.name}-{split_generator.name}{SUFFIX}.{file_format}"""
									UpperCamelCase				:		Any				   =			path_join(self._output_dir						,   _A    )

									UpperCamelCase				:		Any				   =			0
									UpperCamelCase				:		List[str]				   =			0
									UpperCamelCase				:		str				   =			0
									UpperCamelCase				:		Union[str, Any]				   =			[]
									UpperCamelCase				:		str				   =			[]

									for task_id, content in self._prepare_split_single(_A						,   _A						,   _A    ):
															(
															    (
															    UpperCamelCase
															)   ,						(
															    UpperCamelCase
															)   ,						(
															    UpperCamelCase
															)   ,						(
															    UpperCamelCase
															)   ,						
															)				:		Dict				   =			content
															if num_bytes > 0:
																					total_num_examples += num_examples
																					total_num_bytes += num_bytes
																					total_shards += num_shards
																					task_id_and_num_shards.append((task_id, num_shards)    )
																					all_shard_lengths.extend(_A    )

									UpperCamelCase				:		str				   =			total_num_examples
									UpperCamelCase				:		Tuple				   =			total_num_bytes

									# should rename everything at the end
									logger.debug(f"""Renaming {total_shards} shards."""    )
									if total_shards > 1:
															UpperCamelCase				:		List[Any]				   =			all_shard_lengths

															# Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
															# pickling error due to pickling the SparkContext.
															UpperCamelCase				:		List[Any]				   =			self._fs

															# use the -SSSSS-of-NNNNN pattern
															def _rename_shard(
															    _A						,   _A						,   _A						,   ):
																					rename(
																					    _A						,   fpath.replace("""SSSSS"""						,   f"""{shard_id:05d}"""    ).replace("""TTTTT"""						,   f"""{task_id:05d}"""    )						,   fpath.replace("""TTTTT-SSSSS"""						,   f"""{global_shard_id:05d}"""    ).replace("""NNNNN"""						,   f"""{total_shards:05d}"""    )						,   )

															UpperCamelCase				:		str				   =			[]
															UpperCamelCase				:		List[Any]				   =			0
															for i in range(len(_A    )    ):
																					UpperCamelCase   ,						UpperCamelCase				:		Dict				   =			task_id_and_num_shards[i]
																					for shard_id in range(_A    ):
																											args.append([task_id, shard_id, global_shard_id]    )
																											global_shard_id += 1
															self._spark.sparkContext.parallelize(_A						,   len(_A    )    ).map(lambda _A    : _rename_shard(*_A    )    ).collect()
									else:
															# don't use any pattern
															UpperCamelCase				:		int				   =			0
															UpperCamelCase				:		Any				   =			task_id_and_num_shards[0][0]
															self._rename(
															    fpath.replace("""SSSSS"""						,   f"""{shard_id:05d}"""    ).replace("""TTTTT"""						,   f"""{task_id:05d}"""    )						,   fpath.replace(_A						,   """"""    )						,   )







			def   _a      (      self						,   _A						,   ):






									'''simple docstring'''







									return SparkExamplesIterable(self.df    )











'''simple docstring'''
import builtins
import sys

from ...utils.imports import _is_package_available
from . import cursor, input
from .helpers import Direction, clear_line, forceWrite, linebreak, move_cursor, reset_cursor, writeColor
from .keymap import KEYMAP


UpperCamelCase__			=      False
try:
 UpperCamelCase__			=      _is_package_available('google.colab')
except ModuleNotFoundError:
 pass


@input.register
class 							_UpperCAmelCase    :




    def __init__(      self	:							List[Any]	,  a	:							str = None	,  a	:							list = []      ):






       '''simple docstring'''






       lowercase_						:     Optional[Any]							     =  0
       lowercase_						:     Dict							     =  choices
       lowercase_						:     List[str]							     =  prompt
       if sys.platform == "win32":
          lowercase_						:     List[Any]							     =  "*"
       else:
          lowercase_						:     Union[str, Any]							     =  "➔ "




    def 					lowerCAmelCase__					(      self	:							int	,  a	:							Optional[int]	,  a	:							str = ""      ):






       '''simple docstring'''






       if sys.platform != "win32":
          writeColor(self.choices[index]	,  3_2	,  a      )
       else:
          forceWrite(self.choices[index]	,  a      )




    def 					lowerCAmelCase__					(      self	:							List[Any]	,  a	:							int      ):






       '''simple docstring'''






       if index == self.position:
          forceWrite(f""" {self.arrow_char} """      )
          self.write_choice(a      )
       else:
          forceWrite(f"""    {self.choices[index]}"""      )
       reset_cursor()




    def 					lowerCAmelCase__					(      self	:							str	,  a	:							Direction	,  a	:							int = 1      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  self.position
       if direction == Direction.DOWN:
          if self.position + 1 >= len(self.choices      ):
             return
          self.position += num_spaces
       else:
          if self.position - 1 < 0:
             return
          self.position -= num_spaces
       clear_line()
       self.print_choice(a      )
       move_cursor(a	,  direction.name      )
       self.print_choice(self.position      )




    @input.mark(KEYMAP["up"]      )
    def 					lowerCAmelCase__					(      self	:							Dict      ):






       '''simple docstring'''






       self.move_direction(Direction.UP      )




    @input.mark(KEYMAP["down"]      )
    def 					lowerCAmelCase__					(      self	:							List[Any]      ):






       '''simple docstring'''






       self.move_direction(Direction.DOWN      )




    @input.mark(KEYMAP["newline"]      )
    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       move_cursor(len(self.choices      ) - self.position	,  "DOWN"      )
       return self.position




    @input.mark(KEYMAP["interrupt"]      )
    def 					lowerCAmelCase__					(      self	:							str      ):






       '''simple docstring'''






       move_cursor(len(self.choices      ) - self.position	,  "DOWN"      )
       raise KeyboardInterrupt




    @input.mark_multiple(*[KEYMAP[str(a      )] for number in range(1_0      )]      )
    def 					lowerCAmelCase__					(      self	:							int      ):






       '''simple docstring'''






       lowercase_						:     str							     =  int(chr(self.current_selection      )      )
       lowercase_						:     Tuple							     =  index - self.position
       if index == self.position:
          return
       if index < len(self.choices      ):
          if self.position > index:
             self.move_direction(Direction.UP	,  -movement      )
          elif self.position < index:
             self.move_direction(Direction.DOWN	,  a      )
          else:
             return
       else:
          return





    def 					lowerCAmelCase__					(      self	:							Tuple	,  a	:							int = 0      ):






       '''simple docstring'''






       if self.prompt:
          linebreak()
          forceWrite(self.prompt	,  "\n"      )
          if in_colab:
             forceWrite("Please input a choice index (starting from 0), and press enter"	,  "\n"      )
          else:
             forceWrite("Please select a choice using the arrow or number keys, and selecting with enter"	,  "\n"      )
       lowercase_						:     Dict							     =  default_choice
       for i in range(len(self.choices      )      ):
          self.print_choice(a      )
          forceWrite("\n"      )
       move_cursor(len(self.choices      ) - self.position	,  "UP"      )
       with cursor.hide():
          while True:
             if in_colab:
                try:
                   lowercase_						:     List[str]							     =  int(builtins.input()      )
                except ValueError:
                   lowercase_						:     List[Any]							     =  default_choice
             else:
                lowercase_						:     Dict							     =  self.handle_input()
             if choice is not None:
                reset_cursor()
                for _ in range(len(self.choices      ) + 1      ):
                   move_cursor(1	,  "UP"      )
                   clear_line()
                self.write_choice(a	,  "\n"      )
                return choice




