
'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)  ->							Dict:


		'''simple docstring'''


		snake_case  :       Dict     			=						[]
		for part_id in partition_order:
				snake_case  :       Dict     			=						df.where(F'SPARK_PARTITION_ID() = {part_id}'			).collect()
				for row_idx, row in enumerate(SCREAMING_SNAKE_CASE__			):
						expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())			)
		return expected_row_ids_and_row_dicts




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							int:


		'''simple docstring'''


		snake_case  :       List[str]     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       str     			=						spark.range(100			).repartition(1			)
		snake_case  :       Any     			=						Spark(SCREAMING_SNAKE_CASE__			)
		# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
		# that each partition can hold 2 rows.
		spark_builder._repartition_df_if_needed(max_shard_size=16			)
		# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
		assert spark_builder.df.rdd.getNumPartitions() == 50




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							List[Any]:


		'''simple docstring'''


		snake_case  :       List[str]     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       Union[str, Any]     			=						spark.range(10			).repartition(2			)
		snake_case  :       int     			=						[1, 0]
		snake_case  :       List[str]     			=						_generate_iterable_examples(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)  # Reverse the partitions.
		snake_case  :       Optional[int]     			=						_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)

		for i, (row_id, row_dict) in enumerate(generate_fn()			):
				snake_case			,snake_case  :       Optional[int]     			=						expected_row_ids_and_row_dicts[i]
				assert row_id == expected_row_id
				assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							Union[str, Any]:


		'''simple docstring'''


		snake_case  :       Any     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       List[Any]     			=						spark.range(10			).repartition(1			)
		snake_case  :       Optional[Any]     			=						SparkExamplesIterable(SCREAMING_SNAKE_CASE__			)
		assert it.n_shards == 1
		for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__			):
				assert row_id == F'0_{i}'
				assert row_dict == {"id": i}




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							Optional[Any]:


		'''simple docstring'''


		snake_case  :       Optional[int]     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       Tuple     			=						spark.range(30			).repartition(3			)
		# Mock the generator so that shuffle reverses the partition indices.
		with patch('''numpy.random.Generator'''			) as generator_mock:
				snake_case  :       Tuple     			=						lambda SCREAMING_SNAKE_CASE__			: x.reverse()
				snake_case  :       int     			=						_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__ ,       [2, 1, 0]			)

				snake_case  :       List[Any]     			=						SparkExamplesIterable(SCREAMING_SNAKE_CASE__			).shuffle_data_sources(SCREAMING_SNAKE_CASE__			)
				assert shuffled_it.n_shards == 3
				for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__			):
						snake_case			,snake_case  :       Tuple     			=						expected_row_ids_and_row_dicts[i]
						assert row_id == expected_row_id
						assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							int:


		'''simple docstring'''


		snake_case  :       Union[str, Any]     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       Optional[int]     			=						spark.range(20			).repartition(4			)

		# Partitions 0 and 2
		snake_case  :       Optional[int]     			=						SparkExamplesIterable(SCREAMING_SNAKE_CASE__			).shard_data_sources(worker_id=0 ,       num_workers=2			)
		assert shard_it_a.n_shards == 2
		snake_case  :       Tuple     			=						_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__ ,       [0, 2]			)
		for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__			):
				snake_case			,snake_case  :       List[str]     			=						expected_row_ids_and_row_dicts_a[i]
				assert row_id == expected_row_id
				assert row_dict == expected_row_dict

		# Partitions 1 and 3
		snake_case  :       int     			=						SparkExamplesIterable(SCREAMING_SNAKE_CASE__			).shard_data_sources(worker_id=1 ,       num_workers=2			)
		assert shard_it_a.n_shards == 2
		snake_case  :       Any     			=						_get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__ ,       [1, 3]			)
		for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__			):
				snake_case			,snake_case  :       str     			=						expected_row_ids_and_row_dicts_a[i]
				assert row_id == expected_row_id
				assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def    _UpperCamelCase						(       )  ->							List[str]:


		'''simple docstring'''


		snake_case  :       List[Any]     			=						pyspark.sql.SparkSession.builder.master('''local[*]'''			).appName('''pyspark'''			).getOrCreate()
		snake_case  :       int     			=						spark.range(100			).repartition(1			)
		snake_case  :       str     			=						Spark(SCREAMING_SNAKE_CASE__			)
		# Choose a small max_shard_size for maximum partitioning.
		spark_builder._repartition_df_if_needed(max_shard_size=1			)
		# The new number of partitions should not be greater than the number of rows.
		assert spark_builder.df.rdd.getNumPartitions() == 100




'''simple docstring'''
from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)


