






import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__a   :							Tuple						= "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
__a   :							str						= direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__a   :							Optional[Any]						= re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
__a   :							int						= re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__a   :							List[str]						= re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__a   :							Dict						= [
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Optional[int]     )    ->   Optional[Any]:





    """simple docstring"""

    __A       =						re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""      ,					__lowercase     )
    return [m.group(0     ) for m in matches]



def       _SCREAMING_SNAKE_CASE    (			)    ->   Union[str, Any]:





    """simple docstring"""

    __A       =						transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    __A       =						{
        config.replace("""Config"""      ,					""""""     ): model_type for model_type, config in config_maping_names.items()
    }

    # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
    __A       =						collections.defaultdict(__lowercase     )
    __A       =						collections.defaultdict(__lowercase     )
    __A       =						collections.defaultdict(__lowercase     )

    # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
    for attr_name in dir(__lowercase     ):
        __A       =						None
        if _re_tf_models.match(__lowercase     ) is not None:
            __A       =						tf_models
            __A       =						_re_tf_models.match(__lowercase     ).groups()[0]
        elif _re_flax_models.match(__lowercase     ) is not None:
            __A       =						flax_models
            __A       =						_re_flax_models.match(__lowercase     ).groups()[0]
        elif _re_pt_models.match(__lowercase     ) is not None:
            __A       =						pt_models
            __A       =						_re_pt_models.match(__lowercase     ).groups()[0]

        if lookup_dict is not None:
            while len(__lowercase     ) > 0:
                if attr_name in model_prefix_to_model_type:
                    __A       =						True
                    break
                # Try again after removing the last word in the name
                __A       =						"""""".join(camel_case_split(__lowercase     )[:-1]     )

    __A       =						set(list(pt_models.keys()     ) + list(tf_models.keys()     ) + list(flax_models.keys()     )     )
    __A       =						list(__lowercase     )
    all_models.sort()

    __A       =						{"""model_type""": all_models}
    __A       =						[pt_models[t] for t in all_models]
    __A       =						[tf_models[t] for t in all_models]
    __A       =						[flax_models[t] for t in all_models]

    # Now let's use the auto-mapping names to make sure
    __A       =						{}
    for t in all_models:
        if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
            __A       =						"""AutoProcessor"""
        elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
            __A       =						"""AutoTokenizer"""
        elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
            __A       =						"""AutoFeatureExtractor"""
        else:
            # Default to AutoTokenizer if a model has nothing, for backward compatibility.
            __A       =						"""AutoTokenizer"""

    __A       =						[processors[t] for t in all_models]

    return pd.DataFrame(__lowercase     )



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Any     )    ->   Union[str, Any]:





    """simple docstring"""

    __A       =						[
        transformers_module.models.auto.modeling_auto,
        transformers_module.models.auto.modeling_tf_auto,
        transformers_module.models.auto.modeling_flax_auto,
    ]
    for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
        __A       =						[model_mapping, f"TF_{model_mapping}", f"FLAX_{model_mapping}"]
        __A       =						[auto_class, f"TF_{auto_class}", f"Flax_{auto_class}"]
        # Loop through all three frameworks
        for module, cls, mapping in zip(__lowercase      ,					__lowercase      ,					__lowercase     ):
            # The type of pipeline may not exist in this framework
            if not hasattr(__lowercase      ,					__lowercase     ):
                continue
            # First extract all model_names
            __A       =						[]
            for name in getattr(__lowercase      ,					__lowercase     ).values():
                if isinstance(__lowercase      ,					__lowercase     ):
                    model_names.append(__lowercase     )
                else:
                    model_names.extend(list(__lowercase     )     )

            # Add pipeline tag and auto model class for those models
            table.update({model_name: (pipeline_tag, cls) for model_name in model_names}     )

    return table



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Any      ,					__lowercase :      List[Any]     )    ->   Dict:





    """simple docstring"""

    __A       =						get_frameworks_table()
    __A       =						Dataset.from_pandas(__lowercase     )

    __A       =						hf_hub_download(
        """huggingface/transformers-metadata"""      ,					"""pipeline_tags.json"""      ,					repo_type="""dataset"""      ,					token=__lowercase     )
    __A       =						Dataset.from_json(__lowercase     )
    __A       =						{
        tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
        for i in range(len(__lowercase     )     )
    }
    __A       =						update_pipeline_and_auto_class_table(__lowercase     )

    # Sort the model classes to avoid some nondeterministic updates to create false update commits.
    __A       =						sorted(table.keys()     )
    __A       =						pd.DataFrame(
        {
            """model_class""": model_classes,
            """pipeline_tag""": [table[m][0] for m in model_classes],
            """auto_class""": [table[m][1] for m in model_classes],
        }     )
    __A       =						Dataset.from_pandas(__lowercase     )

    with tempfile.TemporaryDirectory() as tmp_dir:
        frameworks_dataset.to_json(os.path.join(__lowercase      ,					"""frameworks.json"""     )     )
        tags_dataset.to_json(os.path.join(__lowercase      ,					"""pipeline_tags.json"""     )     )

        if commit_sha is not None:
            __A       =						(
                f"Update with commit {commit_sha}\n\nSee: "
                f"https://github.com/huggingface/transformers/commit/{commit_sha}"
            )
        else:
            __A       =						"""Update"""

        upload_folder(
            repo_id="""huggingface/transformers-metadata"""      ,					folder_path=__lowercase      ,					repo_type="""dataset"""      ,					token=__lowercase      ,					commit_message=__lowercase      ,					)



def       _SCREAMING_SNAKE_CASE    (			)    ->   Optional[int]:





    """simple docstring"""

    __A       =						{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
    __A       =						transformers_module.pipelines.SUPPORTED_TASKS
    __A       =						[]
    for key in pipeline_tasks:
        if key not in in_table:
            __A       =						pipeline_tasks[key]["""pt"""]
            if isinstance(__lowercase      ,					(list, tuple)     ):
                __A       =						model[0]
            __A       =						model.__name__
            if model not in in_table.values():
                missing.append(__lowercase     )

    if len(__lowercase     ) > 0:
        __A       =						""", """.join(__lowercase     )
        raise ValueError(
            """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
            f"`utils/update_metadata.py`: {msg}. Please add them!"     )


if __name__ == "__main__":
  __a   :							Optional[Any]						= argparse.ArgumentParser()
  parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
  parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
  parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
  __a   :							List[str]						= parser.parse_args()

  if args.check_only:
    check_pipeline_tags()
  else:
    update_metadata(args.token, args.commit_sha)












# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from ..utils import cached_file


# docstyle-ignore
__a   :							Tuple						= "\nHuman: <<task>>\n\nAssistant: "


__a   :							Union[str, Any]						= "huggingface-tools/default-prompts"
__a   :							Optional[int]						= {"chat": "chat_prompt_template.txt", "run": "run_prompt_template.txt"}



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Tuple      ,					__lowercase :      Optional[Any]      ,					__lowercase :      List[Any]="run"     )    ->   Optional[int]:





    """simple docstring"""

    if prompt_or_repo_id is None:
        __A       =						DEFAULT_PROMPTS_REPO

    # prompt is considered a repo ID when it does not contain any kind of space
    if re.search("""\\s"""      ,					__lowercase     ) is not None:
        return prompt_or_repo_id

    __A       =						cached_file(
        __lowercase      ,					PROMPT_FILES[mode]      ,					repo_type="""dataset"""      ,					user_agent={"""agent""": agent_name}     )
    with open(__lowercase      ,					"""r"""      ,					encoding="""utf-8"""     ) as f:
        return f.read()




