
'''simple docstring'''







import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__lowerCAmelCase         =    """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
__lowerCAmelCase         =    direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__lowerCAmelCase         =    re.compile(r"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
__lowerCAmelCase         =    re.compile(r"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__lowerCAmelCase         =    re.compile(r"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__lowerCAmelCase         =    [
    ("""pretraining""", """MODEL_FOR_PRETRAINING_MAPPING_NAMES""", """AutoModelForPreTraining"""),
    ("""feature-extraction""", """MODEL_MAPPING_NAMES""", """AutoModel"""),
    ("""audio-classification""", """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForAudioClassification"""),
    ("""text-generation""", """MODEL_FOR_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForCausalLM"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_CTC_MAPPING_NAMES""", """AutoModelForCTC"""),
    ("""image-classification""", """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForImageClassification"""),
    ("""image-segmentation""", """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES""", """AutoModelForImageSegmentation"""),
    ("""fill-mask""", """MODEL_FOR_MASKED_LM_MAPPING_NAMES""", """AutoModelForMaskedLM"""),
    ("""object-detection""", """MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES""", """AutoModelForObjectDetection"""),
    (
        """zero-shot-object-detection""",
        """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES""",
        """AutoModelForZeroShotObjectDetection""",
    ),
    ("""question-answering""", """MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES""", """AutoModelForQuestionAnswering"""),
    ("""text2text-generation""", """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForSeq2SeqLM"""),
    ("""text-classification""", """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForSequenceClassification"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES""", """AutoModelForSpeechSeq2Seq"""),
    (
        """table-question-answering""",
        """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForTableQuestionAnswering""",
    ),
    ("""token-classification""", """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForTokenClassification"""),
    ("""multiple-choice""", """MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES""", """AutoModelForMultipleChoice"""),
    (
        """next-sentence-prediction""",
        """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES""",
        """AutoModelForNextSentencePrediction""",
    ),
    (
        """audio-frame-classification""",
        """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForAudioFrameClassification""",
    ),
    ("""audio-xvector""", """MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES""", """AutoModelForAudioXVector"""),
    (
        """document-question-answering""",
        """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForDocumentQuestionAnswering""",
    ),
    (
        """visual-question-answering""",
        """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForVisualQuestionAnswering""",
    ),
    ("""image-to-text""", """MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES""", """AutoModelForVision2Seq"""),
    (
        """zero-shot-image-classification""",
        """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForZeroShotImageClassification""",
    ),
    ("""depth-estimation""", """MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES""", """AutoModelForDepthEstimation"""),
    ("""video-classification""", """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForVideoClassification"""),
    ("""mask-generation""", """MODEL_FOR_MASK_GENERATION_MAPPING_NAMES""", """AutoModelForMaskGeneration"""),
]





def 					UpperCAmelCase_		(__a      :    Union[str, Any] ):






   """simple docstring"""

   _a				:	Optional[Any]			=   re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)' ,	__a )
   return [m.group(0 ) for m in matches]





def 					UpperCAmelCase_		():






   """simple docstring"""

   _a				:	Tuple			=   transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
   _a				:	Optional[Any]			=   {
       config.replace('Config' ,	'' ): model_type for model_type, config in config_maping_names.items()
   }

   # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
   _a				:	int			=   collections.defaultdict(__a )
   _a				:	Tuple			=   collections.defaultdict(__a )
   _a				:	Any			=   collections.defaultdict(__a )

   # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
   for attr_name in dir(__a ):
      _a				:	List[str]			=   None
      if _re_tf_models.match(__a ) is not None:
         _a				:	Tuple			=   tf_models
         _a				:	str			=   _re_tf_models.match(__a ).groups()[0]
      elif _re_flax_models.match(__a ) is not None:
         _a				:	Optional[Any]			=   flax_models
         _a				:	Union[str, Any]			=   _re_flax_models.match(__a ).groups()[0]
      elif _re_pt_models.match(__a ) is not None:
         _a				:	int			=   pt_models
         _a				:	Any			=   _re_pt_models.match(__a ).groups()[0]

      if lookup_dict is not None:
         while len(__a ) > 0:
            if attr_name in model_prefix_to_model_type:
               _a				:	int			=   True
               break
            # Try again after removing the last word in the name
            _a				:	Optional[int]			=   ''.join(camel_case_split(__a )[:-1] )

   _a				:	Tuple			=   set(list(pt_models.keys() ) + list(tf_models.keys() ) + list(flax_models.keys() ) )
   _a				:	Union[str, Any]			=   list(__a )
   all_models.sort()

   _a				:	str			=   {'model_type': all_models}
   _a				:	Tuple			=   [pt_models[t] for t in all_models]
   _a				:	Tuple			=   [tf_models[t] for t in all_models]
   _a				:	Dict			=   [flax_models[t] for t in all_models]

   # Now let's use the auto-mapping names to make sure
   _a				:	List[str]			=   {}
   for t in all_models:
      if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
         _a				:	Union[str, Any]			=   'AutoProcessor'
      elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
         _a				:	Dict			=   'AutoTokenizer'
      elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
         _a				:	Union[str, Any]			=   'AutoFeatureExtractor'
      else:
         # Default to AutoTokenizer if a model has nothing, for backward compatibility.
         _a				:	Optional[Any]			=   'AutoTokenizer'

   _a				:	Optional[Any]			=   [processors[t] for t in all_models]

   return pd.DataFrame(__a )





def 					UpperCAmelCase_		(__a      :    Optional[Any] ):






   """simple docstring"""

   _a				:	List[Any]			=   [
       transformers_module.models.auto.modeling_auto,
       transformers_module.models.auto.modeling_tf_auto,
       transformers_module.models.auto.modeling_flax_auto,
   ]
   for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
      _a				:	Dict			=   [model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
      _a				:	str			=   [auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
      # Loop through all three frameworks
      for module, cls, mapping in zip(__a ,	__a ,	__a ):
         # The type of pipeline may not exist in this framework
         if not hasattr(__a ,	__a ):
            continue
         # First extract all model_names
         _a				:	int			=   []
         for name in getattr(__a ,	__a ).values():
            if isinstance(__a ,	__a ):
               model_names.append(__a )
            else:
               model_names.extend(list(__a ) )

            # Add pipeline tag and auto model class for those models
         table.update({model_name: (pipeline_tag, cls) for model_name in model_names} )

   return table





def 					UpperCAmelCase_		(__a      :    str ,	__a      :    Any ):






   """simple docstring"""

   _a				:	List[str]			=   get_frameworks_table()
   _a				:	Dict			=   Dataset.from_pandas(__a )

   _a				:	List[str]			=   hf_hub_download(
       'huggingface/transformers-metadata' ,	'pipeline_tags.json' ,	repo_type='dataset' ,	token=__a )
   _a				:	List[Any]			=   Dataset.from_json(__a )
   _a				:	Optional[Any]			=   {
       tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
       for i in range(len(__a ) )
   }
   _a				:	Union[str, Any]			=   update_pipeline_and_auto_class_table(__a )

   # Sort the model classes to avoid some nondeterministic updates to create false update commits.
   _a				:	List[str]			=   sorted(table.keys() )
   _a				:	Any			=   pd.DataFrame(
       {
           'model_class': model_classes,
           'pipeline_tag': [table[m][0] for m in model_classes],
           'auto_class': [table[m][1] for m in model_classes],
       } )
   _a				:	List[str]			=   Dataset.from_pandas(__a )

   with tempfile.TemporaryDirectory() as tmp_dir:
      frameworks_dataset.to_json(os.path.join(__a ,	'frameworks.json' ) )
      tags_dataset.to_json(os.path.join(__a ,	'pipeline_tags.json' ) )

      if commit_sha is not None:
         _a				:	str			=   (
             f"""Update with commit {commit_sha}\n\nSee: """
             f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
         )
      else:
         _a				:	Optional[Any]			=   'Update'

      upload_folder(
          repo_id='huggingface/transformers-metadata' ,	folder_path=__a ,	repo_type='dataset' ,	token=__a ,	commit_message=__a ,	)





def 					UpperCAmelCase_		():






   """simple docstring"""

   _a				:	Tuple			=   {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
   _a				:	Optional[Any]			=   transformers_module.pipelines.SUPPORTED_TASKS
   _a				:	Optional[int]			=   []
   for key in pipeline_tasks:
      if key not in in_table:
         _a				:	List[str]			=   pipeline_tasks[key]['pt']
         if isinstance(__a ,	(list, tuple) ):
            _a				:	Any			=   model[0]
         _a				:	Union[str, Any]			=   model.__name__
         if model not in in_table.values():
            missing.append(__a )

   if len(__a ) > 0:
      _a				:	Optional[Any]			=   ', '.join(__a )
      raise ValueError(
          'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
          f"""`utils/update_metadata.py`: {msg}. Please add them!""" )


if __name__ == "__main__":
  __lowerCAmelCase         =    argparse.ArgumentParser()
  parser.add_argument("""--token""", type=str, help="""The token to use to push to the transformers-metadata dataset.""")
  parser.add_argument("""--commit_sha""", type=str, help="""The sha of the commit going with this update.""")
  parser.add_argument("""--check-only""", action="""store_true""", help="""Activate to just check all pipelines are present.""")
  __lowerCAmelCase         =    parser.parse_args()

  if args.check_only:
    check_pipeline_tags()
  else:
    update_metadata(args.token, args.commit_sha)





'''simple docstring'''







import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


__lowerCAmelCase         =    logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''	)
@require_torch
@require_tf
@slow
class UpperCAmelCase__	( unittest.TestCase	):





  """simple docstring"""

  def   __lowercase		(		self      :     str							,_a      :     Path							,_a      :     Union[str, None] = None							,_a      :     Union[List[str], None] = None							,_a      :     Union[str, List[str], None] = None							,_a      :     bool = True							,):


     '''simple docstring'''






     _a				:	Optional[int]			=   [file for file in os.listdir(_a       ) if os.path.isfile(os.path.join(_a							,_a       )       )]

     if identifier is not None:
        _a				:	List[str]			=   [file for file in files if identifier in file]

     if n_identifier is not None:
        if isinstance(_a							,_a       ):
           for n_ in n_identifier:
              _a				:	Tuple			=   [file for file in files if n_ not in file]
        else:
           _a				:	Optional[Any]			=   [file for file in files if n_identifier not in file]

     _a				:	List[str]			=   ignore_files or []
     ignore_files.append('__init__.py'       )
     _a				:	Tuple			=   [file for file in files if file not in ignore_files]

     for file in files:
        # Open all files
        print('Testing'							,_a       )

        if only_modules:
           _a				:	Any			=   file.split('.'       )[0]
           try:
              _a				:	List[str]			=   getattr(_a							,_a       )
              _a				:	int			=   doctest.DocTestSuite(_a       )
              _a				:	Any			=   unittest.TextTestRunner().run(_a       )
              self.assertIs(len(result.failures       )							,0       )
           except AttributeError:
              logger.info(F"""{module_identifier} is not a module."""       )
        else:
           _a				:	Union[str, Any]			=   doctest.testfile(str('..' / directory / file       )							,optionflags=doctest.ELLIPSIS       )
           self.assertIs(result.failed							,0       )


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	int			=   Path('src/transformers'       )
     _a				:	List[Any]			=   'modeling'
     _a				:	Optional[Any]			=   [
         'modeling_ctrl.py',
         'modeling_tf_ctrl.py',
     ]
     self.analyze_directory(_a							,identifier=_a							,ignore_files=_a       )


  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     _a				:	Optional[Any]			=   Path('src/transformers'       )
     _a				:	Optional[Any]			=   'tokenization'
     self.analyze_directory(_a							,identifier=_a       )


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Dict			=   Path('src/transformers'       )
     _a				:	str			=   'configuration'
     self.analyze_directory(_a							,identifier=_a       )


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	Tuple			=   Path('src/transformers'       )
     _a				:	List[Any]			=   ['configuration', 'modeling', 'tokenization']
     self.analyze_directory(_a							,n_identifier=_a       )


  def   __lowercase		(		self      :     Tuple       ):


     '''simple docstring'''






     _a				:	List[Any]			=   Path('docs/source'       )
     _a				:	List[str]			=   ['favicon.ico']
     self.analyze_directory(_a							,ignore_files=_a							,only_modules=_a       )



