





import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows





def lowerCAmelCase__(__snake_case							)				->      str:  # picklable for multiprocessing







	'''simple docstring'''







	return i + 1





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def lowerCAmelCase__()				->      Dict:







	'''simple docstring'''







	with parallel_backend('''spark'''							):
		assert ParallelBackendConfig.backend_name == "spark"

	lowerCamelCase__								=		[1, 2, 3]
	with pytest.raises(__a							):
		with parallel_backend('''unsupported backend'''							):
			map_nested(__a       ,__a       ,num_proc=2							)

	with pytest.raises(__a							):
		with parallel_backend('''unsupported backend'''							):
			map_nested(__a       ,__a       ,num_proc=-1							)





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize('''num_proc'''       ,[2, -1]							)
def lowerCAmelCase__(__snake_case							)				->      Tuple:







	'''simple docstring'''







	lowerCamelCase__								=		[1, 2]
	lowerCamelCase__								=		{'''a''': 1, '''b''': 2}
	lowerCamelCase__								=		{'''a''': [1, 2], '''b''': [3, 4]}
	lowerCamelCase__								=		{'''a''': {'''1''': 1}, '''b''': 2}
	lowerCamelCase__								=		{'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
	lowerCamelCase__								=		[2, 3]
	lowerCamelCase__								=		{'''a''': 2, '''b''': 3}
	lowerCamelCase__								=		{'''a''': [2, 3], '''b''': [4, 5]}
	lowerCamelCase__								=		{'''a''': {'''1''': 2}, '''b''': 3}
	lowerCamelCase__								=		{'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}

	with parallel_backend('''spark'''							):
		assert map_nested(__a       ,__a       ,num_proc=__a							) == expected_map_nested_sa
		assert map_nested(__a       ,__a       ,num_proc=__a							) == expected_map_nested_sa
		assert map_nested(__a       ,__a       ,num_proc=__a							) == expected_map_nested_sa
		assert map_nested(__a       ,__a       ,num_proc=__a							) == expected_map_nested_sa
		assert map_nested(__a       ,__a       ,num_proc=__a							) == expected_map_nested_sa










import argparse
from collections import defaultdict

import yaml


A    :					str =      '''docs/source/en/_toctree.yml'''







def    __lowerCamelCase					(							__a   :str    )		->				List[Any]:






		"""simple docstring"""

		A__        =   defaultdict(__a    )
		A__        =   []
		A__        =   []
		for doc in doc_list:
				if "local" in doc:
						counts[doc["local"]] += 1

				if doc["title"].lower() == "overview":
						overview_doc.append({"""local""": doc["""local"""], """title""": doc["""title"""]}    )
				else:
						new_doc_list.append(__a    )

		A__        =   new_doc_list
		A__        =   [key for key, value in counts.items() if value > 1]

		A__        =   []
		for duplicate_key in duplicates:
				A__        =   list({doc["""title"""] for doc in doc_list if doc["""local"""] == duplicate_key}    )
				if len(__a    ) > 1:
						raise ValueError(
						    F'{duplicate_key} is present several times in the documentation table of content at '
						    """`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the """
						    """others."""    )
				# Only add this once
				new_doc.append({"""local""": duplicate_key, """title""": titles[0]}    )

		# Add none duplicate-keys
		new_doc.extend([doc for doc in doc_list if """local""" not in counts or counts[doc["""local"""]] == 1]    )
		A__        =   sorted(__a    ,     key=lambda __a    : s["title"].lower()    )

		# "overview" gets special treatment and is always first
		if len(__a    ) > 1:
				raise ValueError("""{doc_list} has two 'overview' docs which is not allowed."""    )

		overview_doc.extend(__a    )

		# Sort
		return overview_doc







def    __lowerCamelCase					(							__a   :Any=False    )		->				List[str]:






		"""simple docstring"""

		with open(__a    ,     encoding="""utf-8"""    ) as f:
				A__        =   yaml.safe_load(f.read()    )

		# Get to the API doc
		A__        =   0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		A__        =   content[api_idx]["""sections"""]

		# Then to the model doc
		A__        =   0
		while api_doc[scheduler_idx]["title"] != "Schedulers":
				scheduler_idx += 1

		A__        =   api_doc[scheduler_idx]["""sections"""]
		A__        =   clean_doc_toc(__a    )

		A__        =   False
		if new_scheduler_doc != scheduler_doc:
				A__        =   True
				if overwrite:
						A__        =   new_scheduler_doc

		if diff:
				if overwrite:
						A__        =   api_doc
						with open(__a    ,     """w"""    ,     encoding="""utf-8"""    ) as f:
								f.write(yaml.dump(__a    ,     allow_unicode=__a    )    )
				else:
						raise ValueError(
						    """The model doc part of the table of content is not properly sorted, run `make style` to fix this."""    )







def    __lowerCamelCase					(							__a   :Optional[int]=False    )		->				Dict:






		"""simple docstring"""

		with open(__a    ,     encoding="""utf-8"""    ) as f:
				A__        =   yaml.safe_load(f.read()    )

		# Get to the API doc
		A__        =   0
		while content[api_idx]["title"] != "API":
				api_idx += 1
		A__        =   content[api_idx]["""sections"""]

		# Then to the model doc
		A__        =   0
		while api_doc[pipeline_idx]["title"] != "Pipelines":
				pipeline_idx += 1

		A__        =   False
		A__        =   api_doc[pipeline_idx]["""sections"""]
		A__        =   []

		# sort sub pipeline docs
		for pipeline_doc in pipeline_docs:
				if "section" in pipeline_doc:
						A__        =   pipeline_doc["""section"""]
						A__        =   clean_doc_toc(__a    )
						if overwrite:
								A__        =   new_sub_pipeline_doc
				new_pipeline_docs.append(__a    )

		# sort overall pipeline doc
		A__        =   clean_doc_toc(__a    )

		if new_pipeline_docs != pipeline_docs:
				A__        =   True
				if overwrite:
						A__        =   new_pipeline_docs

		if diff:
				if overwrite:
						A__        =   api_doc
						with open(__a    ,     """w"""    ,     encoding="""utf-8"""    ) as f:
								f.write(yaml.dump(__a    ,     allow_unicode=__a    )    )
				else:
						raise ValueError(
						    """The model doc part of the table of content is not properly sorted, run `make style` to fix this."""    )


if __name__ == "__main__":
					A    :					Tuple =      argparse.ArgumentParser()
					parser.add_argument('''--fix_and_overwrite''', action='''store_true''', help='''Whether to fix inconsistencies.''')
					A    :					Optional[Any] =      parser.parse_args()

					check_scheduler_doc(args.fix_and_overwrite)
					check_pipeline_doc(args.fix_and_overwrite)



