





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam





class  A__    (							datasets.BeamBasedBuilder	):




   """simple docstring"""



   def 							__lowercase		(					self)			->						Optional[int]:







          '''simple docstring'''

          return datasets.DatasetInfo(
              features=datasets.Features({'content': datasets.Value('string')})					,  supervised_keys=lowercase					,  )

   def 							__lowercase		(					self					,  lowercase					,  lowercase)			->						List[str]:







          '''simple docstring'''

          return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,  gen_kwargs={'examples': get_test_dummy_examples()})]

   def 							__lowercase		(					self					,  lowercase					,  lowercase)			->						List[str]:







          '''simple docstring'''

          import apache_beam as beam

          return pipeline | "Load Examples" >> beam.Create(lowercase)





class  A__    (							datasets.BeamBasedBuilder	):




   """simple docstring"""



   def 							__lowercase		(					self)			->						int:







          '''simple docstring'''

          return datasets.DatasetInfo(
              features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})})					,  supervised_keys=lowercase					,  )

   def 							__lowercase		(					self					,  lowercase					,  lowercase)			->						Union[str, Any]:







          '''simple docstring'''

          return [
              datasets.SplitGenerator(name=datasets.Split.TRAIN					,  gen_kwargs={'examples': get_test_nested_examples()})
          ]

   def 							__lowercase		(					self					,  lowercase					,  lowercase)			->						Union[str, Any]:







          '''simple docstring'''

          import apache_beam as beam

          return pipeline | "Load Examples" >> beam.Create(lowercase)



def A_	(		)  ->		List[Any]:
       return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']				)]



def A_	(		)  ->		Dict:
       return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']				)]





class  A__    (							__UpperCAmelCase	):




   """simple docstring"""



   @require_beam
   def 							__lowercase		(					self)			->						Optional[Any]:







          '''simple docstring'''

          a__						:	Optional[Any] =				len(get_test_dummy_examples())
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
                 a__						:	List[str] =				DummyBeamDataset(cache_dir=lowercase					,  beam_runner='DirectRunner')
                 builder.download_and_prepare()
                 self.assertTrue(
                     os.path.exists(
                         os.path.join(lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  F'{builder.name}-train.arrow')))
                 self.assertDictEqual(builder.info.features					,  datasets.Features({'content': datasets.Value('string')}))
                 a__						:	Tuple =				builder.as_dataset()
                 self.assertEqual(dset['train'].num_rows					,  lowercase)
                 self.assertEqual(dset['train'].info.splits['train'].num_examples					,  lowercase)
                 self.assertDictEqual(dset['train'][0]					,  get_test_dummy_examples()[0][1])
                 self.assertDictEqual(
                     dset['train'][expected_num_examples - 1]					,  get_test_dummy_examples()[expected_num_examples - 1][1])
                 self.assertTrue(
                     os.path.exists(os.path.join(lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  'dataset_info.json')))
                 del dset

   @require_beam
   def 							__lowercase		(					self)			->						Tuple:







          '''simple docstring'''

          import apache_beam as beam

          a__						:	int =				beam.io.parquetio.WriteToParquet

          a__						:	Optional[int] =				len(get_test_dummy_examples())
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
                 a__						:	Optional[int] =				DummyBeamDataset(cache_dir=lowercase					,  beam_runner='DirectRunner')
                 with patch('apache_beam.io.parquetio.WriteToParquet') as write_parquet_mock:
                        a__						:	Dict =				partial(lowercase					,  num_shards=2)
                        builder.download_and_prepare()
                 self.assertTrue(
                     os.path.exists(
                         os.path.join(
                             lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  F'{builder.name}-train-00000-of-00002.arrow')))
                 self.assertTrue(
                     os.path.exists(
                         os.path.join(
                             lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  F'{builder.name}-train-00000-of-00002.arrow')))
                 self.assertDictEqual(builder.info.features					,  datasets.Features({'content': datasets.Value('string')}))
                 a__						:	Optional[Any] =				builder.as_dataset()
                 self.assertEqual(dset['train'].num_rows					,  lowercase)
                 self.assertEqual(dset['train'].info.splits['train'].num_examples					,  lowercase)
                 # Order is not preserved when sharding, so we just check that all the elements are there
                 self.assertListEqual(sorted(dset['train']['content'])					,  sorted(['foo', 'bar', 'foobar']))
                 self.assertTrue(
                     os.path.exists(os.path.join(lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  'dataset_info.json')))
                 del dset

   @require_beam
   def 							__lowercase		(					self)			->						Optional[Any]:







          '''simple docstring'''

          with tempfile.TemporaryDirectory() as tmp_cache_dir:
                 a__						:	str =				DummyBeamDataset(cache_dir=lowercase)
                 self.assertRaises(datasets.builder.MissingBeamOptions					,  builder.download_and_prepare)

   @require_beam
   def 							__lowercase		(					self)			->						List[Any]:







          '''simple docstring'''

          a__						:	str =				len(get_test_nested_examples())
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
                 a__						:	Tuple =				NestedBeamDataset(cache_dir=lowercase					,  beam_runner='DirectRunner')
                 builder.download_and_prepare()
                 self.assertTrue(
                     os.path.exists(
                         os.path.join(lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  F'{builder.name}-train.arrow')))
                 self.assertDictEqual(
                     builder.info.features					,  datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string')})}))
                 a__						:	Optional[int] =				builder.as_dataset()
                 self.assertEqual(dset['train'].num_rows					,  lowercase)
                 self.assertEqual(dset['train'].info.splits['train'].num_examples					,  lowercase)
                 self.assertDictEqual(dset['train'][0]					,  get_test_nested_examples()[0][1])
                 self.assertDictEqual(
                     dset['train'][expected_num_examples - 1]					,  get_test_nested_examples()[expected_num_examples - 1][1])
                 self.assertTrue(
                     os.path.exists(os.path.join(lowercase					,  builder.name					,  'default'					,  '0.0.0'					,  'dataset_info.json')))
                 del dset









import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowercase      :       List[str]									=  """src/transformers"""
lowercase      :       Optional[int]									=  """docs/source/en/tasks"""



def A_	(		A__		,      A__		,      A__				)  ->		Tuple:
       with open(A__		,      'r'		,      encoding='utf-8'		,      newline='\n'				) as f:
              a__						:	Any =				f.readlines()
       # Find the start prompt.
       a__						:	str =				0
       while not lines[start_index].startswith(A__				):
              start_index += 1
       start_index += 1

       a__						:	int =				start_index
       while not lines[end_index].startswith(A__				):
              end_index += 1
       end_index -= 1

       while len(lines[start_index]				) <= 1:
              start_index += 1
       while len(lines[end_index]				) <= 1:
              end_index -= 1
       end_index += 1
       return "".join(lines[start_index:end_index]				), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowercase      :       Tuple									=  direct_transformers_import(TRANSFORMERS_PATH)

lowercase      :       Optional[Any]									=  {
    """asr.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    """audio_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    """language_modeling.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    """image_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    """masked_language_modeling.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    """multiple_choice.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    """object_detection.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    """question_answering.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    """semantic_segmentation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    """sequence_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    """summarization.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    """token_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    """translation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    """video_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    """document_question_answering.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    """monocular_depth_estimation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowercase      :       Optional[Any]									=  {
    """summarization.md""": ("""nllb""",),
    """translation.md""": ("""nllb""",),
}



def A_	(		A__				)  ->		Optional[int]:
       a__						:	Optional[Any] =				TASK_GUIDE_TO_MODELS[task_guide]
       a__						:	int =				SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(A__		,      set()				)
       a__						:	Optional[Any] =				{
           code: name
           for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
           if (code in model_maping_names or code in special_model_types)
       }
       return ", ".join([F'[{name}](../model_doc/{code})' for code, name in model_names.items()]				) + "\n"



def A_	(		A__		,      A__=False				)  ->		Optional[int]:
       a__ ,							a__ ,							a__ ,							a__						:	Dict =				_find_text_in_file(
           filename=os.path.join(A__		,      A__				)		,      start_prompt='<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'		,      end_prompt='<!--End of the generated tip-->'		,      )

       a__						:	List[Any] =				get_model_list_for_task(A__				)

       if current_list != new_list:
              if overwrite:
                     with open(os.path.join(A__		,      A__				)		,      'w'		,      encoding='utf-8'		,      newline='\n'				) as f:
                            f.writelines(lines[:start_index] + [new_list] + lines[end_index:]				)
              else:
                     raise ValueError(
                         F'The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'
                         ' to fix this.'				)


if __name__ == "__main__":
    lowercase      :       Dict									=  argparse.ArgumentParser()
    parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
    lowercase      :       str									=  parser.parse_args()

    for task_guide in TASK_GUIDE_TO_MODELS.keys():
        check_model_list_for_task(task_guide, args.fix_and_overwrite)


