"""simple docstring"""




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch







def 				_UpperCAmelCase		(							lowerCamelCase__  ):  # picklable for multiprocessing





		"""simple docstring"""




		return x.sum()







def 				_UpperCAmelCase		(							lowerCamelCase__  ):  # picklable for multiprocessing





		"""simple docstring"""




		return i + 1

@dataclass
class    a_   :
					UpperCamelCase_			:						int
					UpperCamelCase_			:						str

class    a_   (  __UpperCamelCase      ):




					def     _SCREAMING_SNAKE_CASE    (						self    :  List[str]  ):
							lowerCAmelCase__   = {}
							lowerCAmelCase__   = []
							lowerCAmelCase__   = 1
							lowerCAmelCase__   = [1, 2]
							lowerCAmelCase__   = {"""a""": 1, """b""": 2}
							lowerCAmelCase__   = {"""a""": [1, 2], """b""": [3, 4]}
							lowerCAmelCase__   = {"""a""": {"""1""": 1}, """b""": 2}
							lowerCAmelCase__   = {"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
							lowerCAmelCase__   = {}
							lowerCAmelCase__   = []
							lowerCAmelCase__   = 2
							lowerCAmelCase__   = [2, 3]
							lowerCAmelCase__   = {"""a""": 2, """b""": 3}
							lowerCAmelCase__   = {"""a""": [2, 3], """b""": [4, 5]}
							lowerCAmelCase__   = {"""a""": {"""1""": 2}, """b""": 3}
							lowerCAmelCase__   = {"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__  )	,    snake_case__  )

							lowerCAmelCase__   = 2
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )

							lowerCAmelCase__   = {"""a""": np.eye(2  ), """b""": np.zeros(3  ), """c""": np.ones(2  )}
							lowerCAmelCase__   = {"""a""": 2, """b""": 0, """c""": 2}
							lowerCAmelCase__   = {
							    """a""": np.eye(2  ).astype(snake_case__  ),
							    """b""": np.zeros(3  ).astype(snake_case__  ),
							    """c""": np.ones(2  ).astype(snake_case__  ),
							}
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    map_numpy=snake_case__  )	,    snake_case__  )
							self.assertEqual(
							    {k: v.tolist() for k, v in map_nested(snake_case__	,    snake_case__	,    map_numpy=snake_case__  ).items()}	,    {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}	,    )
							self.assertEqual(map_nested(snake_case__	,    snake_case__	,    map_numpy=snake_case__	,    num_proc=snake_case__  )	,    snake_case__  )
							self.assertEqual(
							    {k: v.tolist() for k, v in map_nested(snake_case__	,    snake_case__	,    map_numpy=snake_case__	,    num_proc=snake_case__  ).items()}	,    {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}	,    )
							with self.assertRaises(snake_case__  ):  # can't pickle a local lambda
									map_nested(lambda snake_case__  : x + 1	,    snake_case__	,    num_proc=snake_case__  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  str  ):
							lowerCAmelCase__   = {"""a""": 1, """b""": 2}
							lowerCAmelCase__   = {"""a""": 3, """b""": 4}
							lowerCAmelCase__   = {"""a""": 5, """b""": 6}
							lowerCAmelCase__   = sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]  )
							self.assertEqual(sorted(zip_dict(snake_case__	,    snake_case__	,    snake_case__  )  )	,    snake_case__  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  int  ):

							class    a_   :
												UpperCamelCase_			:						Tuple      =     "bar"
							lowerCAmelCase__   = Foo()
							self.assertEqual(foo.my_attr	,    """bar"""  )
							with temporary_assignment(snake_case__	,    """my_attr"""	,    """BAR"""  ):
									self.assertEqual(foo.my_attr	,    """BAR"""  )
							self.assertEqual(foo.my_attr	,    """bar"""  )







@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""      ,	[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]      ,	)
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		with patch("""datasets.utils.py_utils._single_map_nested"""  ) as mock_single_map_nested, patch(
		    """datasets.parallel.parallel.Pool"""  ) as mock_multiprocessing_pool:
				lowerCAmelCase__   = {f"""{i}""": i for i in range(lowerCamelCase__  )}
				lowerCAmelCase__   = map_nested(lambda lowerCamelCase__  : x + 10      ,	lowerCamelCase__      ,	num_proc=lowerCamelCase__      ,	parallel_min_length=16  )
				if expected_num_proc == 1:
						assert mock_single_map_nested.called
						assert not mock_multiprocessing_pool.called
				else:
						assert not mock_single_map_nested.called
						assert mock_multiprocessing_pool.called
						assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc

class    a_   (  __UpperCamelCase      ):




					@require_tf
					def     _SCREAMING_SNAKE_CASE    (						self    :  Optional[int]  ):
							import tensorflow as tf
							from tensorflow.keras import layers

							lowerCAmelCase__   = layers.Dense(2  )

							def gen_random_output():
									lowerCAmelCase__   = tf.random.uniform((1, 3)  )
									return model(snake_case__  ).numpy()

							with temp_seed(42	,    set_tensorflow=snake_case__  ):
									lowerCAmelCase__   = gen_random_output()
							with temp_seed(42	,    set_tensorflow=snake_case__  ):
									lowerCAmelCase__   = gen_random_output()
							lowerCAmelCase__   = gen_random_output()

							np.testing.assert_equal(snake_case__	,    snake_case__  )
							self.assertGreater(np.abs(outa - outa  ).sum()	,    0  )




					@require_torch
					def     _SCREAMING_SNAKE_CASE    (						self    :  List[str]  ):
							import torch

							def gen_random_output():
									lowerCAmelCase__   = torch.nn.Linear(3	,    2  )
									lowerCAmelCase__   = torch.rand(1	,    3  )
									return model(snake_case__  ).detach().numpy()

							with temp_seed(42	,    set_pytorch=snake_case__  ):
									lowerCAmelCase__   = gen_random_output()
							with temp_seed(42	,    set_pytorch=snake_case__  ):
									lowerCAmelCase__   = gen_random_output()
							lowerCAmelCase__   = gen_random_output()

							np.testing.assert_equal(snake_case__	,    snake_case__  )
							self.assertGreater(np.abs(outa - outa  ).sum()	,    0  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Union[str, Any]  ):
							def gen_random_output():
									return np.random.rand(1	,    3  )

							with temp_seed(42  ):
									lowerCAmelCase__   = gen_random_output()
							with temp_seed(42  ):
									lowerCAmelCase__   = gen_random_output()
							lowerCAmelCase__   = gen_random_output()

							np.testing.assert_equal(snake_case__	,    snake_case__  )
							self.assertGreater(np.abs(outa - outa  ).sum()	,    0  )







@pytest.mark.parametrize("""input_data"""      ,	[{}]  )
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = NestedDataStructure(lowerCamelCase__  ).data
		assert output_data == input_data







@pytest.mark.parametrize(
    """data, expected_output"""      ,	[
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]      ,	)
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = NestedDataStructure(lowerCamelCase__  ).flatten()
		assert output == expected_output







def 				_UpperCAmelCase		(							):





		"""simple docstring"""




		lowerCAmelCase__   = A(x=1      ,	y="""foobar"""  )
		lowerCAmelCase__   = {"""x""": 1, """y""": """foobar"""}
		assert asdict(lowerCamelCase__  ) == expected_output

		lowerCAmelCase__   = {"""a""": {"""b""": A(x=10      ,	y="""foo"""  )}, """c""": [A(x=20      ,	y="""bar"""  )]}
		lowerCAmelCase__   = {"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
		assert asdict(lowerCamelCase__  ) == expected_output

		with pytest.raises(lowerCamelCase__  ):
				asdict([1, A(x=10      ,	y="""foo"""  )]  )







def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		return text.split()







def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		yield (time.time(), content)
		time.sleep(2  )
		yield (time.time(), content)







def 				_UpperCAmelCase		(							):





		"""simple docstring"""




		with Pool(2  ) as pool:
				lowerCAmelCase__   = list(iflatmap_unordered(lowerCamelCase__      ,	_split_text      ,	kwargs_iterable=[{"""text""": """hello there"""}] * 10  )  )
				assert out.count("""hello"""  ) == 10
				assert out.count("""there"""  ) == 10
				assert len(lowerCamelCase__  ) == 20

		# check multiprocess from pathos (uses dill for pickling)
		with multiprocess.Pool(2  ) as pool:
				lowerCAmelCase__   = list(iflatmap_unordered(lowerCamelCase__      ,	_split_text      ,	kwargs_iterable=[{"""text""": """hello there"""}] * 10  )  )
				assert out.count("""hello"""  ) == 10
				assert out.count("""there"""  ) == 10
				assert len(lowerCamelCase__  ) == 20

		# check that we get items as fast as possible
		with Pool(2  ) as pool:
				lowerCAmelCase__   = []
				for yield_time, content in iflatmap_unordered(
				    lowerCamelCase__      ,	_aseconds_generator_of_aitems_with_timing      ,	kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]  ):
						assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
						out.append(lowerCamelCase__  )
				assert out.count("""a"""  ) == 2
				assert out.count("""b"""  ) == 2
				assert len(lowerCamelCase__  ) == 4





"""simple docstring"""




import time
from contextlib import contextmanager
from pathlib import Path

import pytest
import requests
from huggingface_hub.hf_api import HfApi, HfFolder


__lowerCAmelCase     :				str    				=    "__DUMMY_TRANSFORMERS_USER__"
__lowerCAmelCase     :				Dict    				=    "Dummy User"
__lowerCAmelCase     :				Optional[Any]    				=    "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"

__lowerCAmelCase     :				str    				=    "https://hub-ci.huggingface.co"
__lowerCAmelCase     :				int    				=    CI_HUB_ENDPOINT + "/datasets/{repo_id}/resolve/{revision}/{path}"
__lowerCAmelCase     :				Dict    				=    CI_HUB_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
__lowerCAmelCase     :				Optional[Any]    				=    Path("~/.huggingface/hub_ci_token").expanduser()







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr(
		    """huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr("""datasets.config.HF_ENDPOINT"""      ,	lowerCamelCase__  )
		monkeypatch.setattr("""datasets.config.HUB_DATASETS_URL"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		monkeypatch.setattr("""huggingface_hub.hf_api.HfFolder.path_token"""      ,	lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		HfFolder.save_token(lowerCamelCase__  )
		yield
		HfFolder.delete_token()







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							):





		"""simple docstring"""




		return HfApi(endpoint=lowerCamelCase__  )







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = HfFolder.get_token()
		HfFolder.save_token(lowerCamelCase__  )
		yield CI_HUB_USER_TOKEN
		if previous_token is not None:
				HfFolder.save_token(lowerCamelCase__  )







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		def _cleanup_repo(lowerCamelCase__  ):
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )

		return _cleanup_repo







@pytest.fixture
def 				_UpperCAmelCase		(							lowerCamelCase__  ):





		"""simple docstring"""




		@contextmanager
		def _temporary_repo(lowerCamelCase__  ):
				try:
						yield repo_id
				finally:
						cleanup_repo(lowerCamelCase__  )

		return _temporary_repo







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_txt_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data/text_data.txt"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_txt_data_







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_zipped_txt_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data.zip"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_zipped_txt_data_







@pytest.fixture(scope="""session"""  )
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		lowerCAmelCase__   = f"""repo_zipped_img_data-{int(time.time() * 10e3  )}"""
		lowerCAmelCase__   = f"""{CI_HUB_USER}/{repo_name}"""
		hf_api.create_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""      ,	private=lowerCamelCase__  )
		hf_api.upload_file(
		    token=lowerCamelCase__      ,	path_or_fileobj=str(lowerCamelCase__  )      ,	path_in_repo="""data.zip"""      ,	repo_id=lowerCamelCase__      ,	repo_type="""dataset"""      ,	)
		yield repo_id
		try:
				hf_api.delete_repo(lowerCamelCase__      ,	token=lowerCamelCase__      ,	repo_type="""dataset"""  )
		except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
				pass







@pytest.fixture()
def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__  ):





		"""simple docstring"""




		return hf_private_dataset_repo_zipped_img_data_




