


'''simple docstring'''






from typing import List

import numpy as np







def 				UpperCamelCase_			(    A__					:   dict					):
  '''simple docstring'''





  lowerCAmelCase_			:      Optional[Any]			=   {key: len(A__					) for key, value in gen_kwargs.items() if isinstance(A__							,    A__					)}
  if len(set(lists_lengths.values()					)					) > 1:
    raise RuntimeError(
        (
            """Sharding is ambiguous for this dataset: """
            + """we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n"""
            + """\n""".join(f'\t- key {key} has length {length}' for key, length in lists_lengths.items()					)
            + """\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, """
            + """and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."""
        )					)
  lowerCAmelCase_			:      Tuple			=   max(lists_lengths.values()							,    default=0					)
  return max(1							,    A__					)







def 				UpperCamelCase_			(    A__					:   int							,    A__					:   int					):
  '''simple docstring'''





  lowerCAmelCase_			:      Union[str, Any]			=   []
  for group_idx in range(A__					):
    lowerCAmelCase_			:      List[str]			=   num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
    if num_shards_to_add == 0:
      break
    lowerCAmelCase_			:      List[str]			=   shards_indices_per_group[-1].stop if shards_indices_per_group else 0
    lowerCAmelCase_			:      Any			=   range(A__							,    start + num_shards_to_add					)
    shards_indices_per_group.append(A__					)
  return shards_indices_per_group







def 				UpperCamelCase_			(    A__					:   dict							,    A__					:   int					):
  '''simple docstring'''





  lowerCAmelCase_			:      Optional[int]			=   _number_of_shards_in_gen_kwargs(A__					)
  if num_shards == 1:
    return [dict(A__					)]
  else:
    lowerCAmelCase_			:      Tuple			=   _distribute_shards(num_shards=A__							,    max_num_jobs=A__					)
    return [
        {
            key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
            if isinstance(A__							,    A__					)
            else value
            for key, value in gen_kwargs.items()
        }
        for group_idx in range(len(A__					)					)
    ]







def 				UpperCamelCase_			(    A__					:   List[dict]					):
  '''simple docstring'''





  return {
      key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
      if isinstance(gen_kwargs_list[0][key]							,    A__					)
      else gen_kwargs_list[0][key]
      for key in gen_kwargs_list[0]
  }







def 				UpperCamelCase_			(    A__					:   np.random.Generator							,    A__					:   dict					):
  '''simple docstring'''





  lowerCAmelCase_			:      List[str]			=   {len(A__					) for value in gen_kwargs.values() if isinstance(A__							,    A__					)}
  lowerCAmelCase_			:      List[str]			=   {}
  for size in list_sizes:
    lowerCAmelCase_			:      Optional[int]			=   list(range(A__					)					)
    rng.shuffle(indices_per_size[size]					)
  # Now let's copy the gen_kwargs and shuffle the lists based on their sizes
  lowerCAmelCase_			:      int			=   dict(A__					)
  for key, value in shuffled_kwargs.items():
    if isinstance(A__							,    A__					):
      lowerCAmelCase_			:      int			=   [value[i] for i in indices_per_size[len(A__					)]]
  return shuffled_kwargs







'''simple docstring'''






import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoFeatureExtractor, WavaVecaFeatureExtractor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_feature_extraction import CustomFeatureExtractor  # noqa E402


__A					:  str											=      get_tests_dir("fixtures")






class    __snake_case       (   unittest.TestCase):







    """simple docstring"""



    def __lowercase   (   self    :  int		)   ->  List[str]:
      # A mock response for an HTTP head request to emulate server down
      lowerCAmelCase_			:      str			=   mock.Mock()
      lowerCAmelCase_			:      Optional[Any]			=   5_00
      lowerCAmelCase_			:      List[str]			=   {}
      lowerCAmelCase_			:      str			=   HTTPError
      lowerCAmelCase_			:      Any			=   {}

      # Download this model to make sure it's in the cache.
      lowerCAmelCase_			:      Optional[Any]			=   WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""		)
      # Under the mock environment we get a 500 error when trying to reach the model.
      with mock.patch("""requests.Session.request"""    ,						return_value=lowerCamelCase		) as mock_head:
        lowerCAmelCase_			:      Union[str, Any]			=   WavaVecaFeatureExtractor.from_pretrained("""hf-internal-testing/tiny-random-wav2vec2"""		)
        # This check we did call the fake head request
        mock_head.assert_called()





    def __lowercase   (   self    :  Dict		)   ->  Any:
      # This test is for deprecated behavior and can be removed in v5
      lowerCAmelCase_			:      List[str]			=   WavaVecaFeatureExtractor.from_pretrained(
          """https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json"""		)







@is_staging_test
class    __snake_case       (   unittest.TestCase):







    """simple docstring"""



    @classmethod
    def __lowercase   (   cls    :  Tuple		)   ->  str:
      lowerCAmelCase_			:      Dict			=   TOKEN
      HfFolder.save_token(lowerCamelCase		)

    @classmethod
    def __lowercase   (   cls    :  Any		)   ->  Any:
      try:
        delete_repo(token=cls._token    ,						repo_id="""test-feature-extractor"""		)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token    ,						repo_id="""valid_org/test-feature-extractor-org"""		)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token    ,						repo_id="""test-dynamic-feature-extractor"""		)
      except HTTPError:
        pass

    def __lowercase   (   self    :  int		)   ->  str:
      lowerCAmelCase_			:      Tuple			=   WavaVecaFeatureExtractor.from_pretrained(lowerCamelCase		)
      feature_extractor.push_to_hub("""test-feature-extractor"""    ,						use_auth_token=self._token		)

      lowerCAmelCase_			:      Optional[Any]			=   WavaVecaFeatureExtractor.from_pretrained(F'{USER}/test-feature-extractor'		)
      for k, v in feature_extractor.__dict__.items():
        self.assertEqual(lowerCamelCase    ,						getattr(lowerCamelCase    ,						lowerCamelCase		)		)

      # Reset repo
      delete_repo(token=self._token    ,						repo_id="""test-feature-extractor"""		)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        feature_extractor.save_pretrained(
            lowerCamelCase    ,						repo_id="""test-feature-extractor"""    ,						push_to_hub=lowerCamelCase    ,						use_auth_token=self._token		)

      lowerCAmelCase_			:      Any			=   WavaVecaFeatureExtractor.from_pretrained(F'{USER}/test-feature-extractor'		)
      for k, v in feature_extractor.__dict__.items():
        self.assertEqual(lowerCamelCase    ,						getattr(lowerCamelCase    ,						lowerCamelCase		)		)

    def __lowercase   (   self    :  Optional[Any]		)   ->  int:
      lowerCAmelCase_			:      Union[str, Any]			=   WavaVecaFeatureExtractor.from_pretrained(lowerCamelCase		)
      feature_extractor.push_to_hub("""valid_org/test-feature-extractor"""    ,						use_auth_token=self._token		)

      lowerCAmelCase_			:      Optional[int]			=   WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor"""		)
      for k, v in feature_extractor.__dict__.items():
        self.assertEqual(lowerCamelCase    ,						getattr(lowerCamelCase    ,						lowerCamelCase		)		)

      # Reset repo
      delete_repo(token=self._token    ,						repo_id="""valid_org/test-feature-extractor"""		)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        feature_extractor.save_pretrained(
            lowerCamelCase    ,						repo_id="""valid_org/test-feature-extractor-org"""    ,						push_to_hub=lowerCamelCase    ,						use_auth_token=self._token		)

      lowerCAmelCase_			:      Optional[int]			=   WavaVecaFeatureExtractor.from_pretrained("""valid_org/test-feature-extractor-org"""		)
      for k, v in feature_extractor.__dict__.items():
        self.assertEqual(lowerCamelCase    ,						getattr(lowerCamelCase    ,						lowerCamelCase		)		)





    def __lowercase   (   self    :  Optional[Any]		)   ->  Any:
      CustomFeatureExtractor.register_for_auto_class()
      lowerCAmelCase_			:      Dict			=   CustomFeatureExtractor.from_pretrained(lowerCamelCase		)

      feature_extractor.push_to_hub("""test-dynamic-feature-extractor"""    ,						use_auth_token=self._token		)

      # This has added the proper auto_map field to the config
      self.assertDictEqual(
          feature_extractor.auto_map    ,						{"""AutoFeatureExtractor""": """custom_feature_extraction.CustomFeatureExtractor"""}    ,						)

      lowerCAmelCase_			:      Union[str, Any]			=   AutoFeatureExtractor.from_pretrained(
          F'{USER}/test-dynamic-feature-extractor'    ,						trust_remote_code=lowerCamelCase		)
      # Can't make an isinstance check because the new_feature_extractor is from the CustomFeatureExtractor class of a dynamic module
      self.assertEqual(new_feature_extractor.__class__.__name__    ,						"""CustomFeatureExtractor"""		)



