






"""simple docstring"""

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase__          = logging.get_logger(__name__)

lowercase__          = "▁"

lowercase__          = {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase__          = {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

lowercase__          = {
    "facebook/m2m100_418M": 1024,
}

# fmt: off
lowercase__          = {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}


class 		__lowerCamelCase				( SCREAMING_SNAKE_CASE__			):





		'''simple docstring'''






		a_	:				List[Any]            =							VOCAB_FILES_NAMES
		a_	:				Dict            =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_	:				Optional[int]            =							PRETRAINED_VOCAB_FILES_MAP
		a_	:				Optional[Any]            =							["""input_ids""", """attention_mask"""]

		a_	:				int            =							[]
		a_	:				Optional[Any]            =							[]





		def __init__(   self							:	str						,			a_							:	List[Any]						,			a_							:	int						,			a_							:	Optional[Any]=None						,			a_							:	int=None						,			a_							:	Dict="<s>"						,			a_							:	Union[str, Any]="</s>"						,			a_							:	List[Any]="</s>"						,			a_							:	int="<pad>"						,			a_							:	Optional[Any]="<unk>"						,			a_							:	Union[str, Any]="m2m100"						,			a_							:	Optional[Dict[str, Any]] = None						,			a_							:	Union[str, Any]=8						,			**a_							:	List[Any]						,			):
				lowerCAmelCase_   :							Optional[Any]									=			{} if sp_model_kwargs is None else sp_model_kwargs

				lowerCAmelCase_   :							Optional[int]									=			language_codes
				lowerCAmelCase_   :							Dict									=			FAIRSEQ_LANGUAGE_CODES[language_codes]
				lowerCAmelCase_   :							Any									=			{lang_code: f'''__{lang_code}__''' for lang_code in fairseq_language_code}

				lowerCAmelCase_   :							Optional[Any]									=			kwargs.get("additional_special_tokens"						,			[]	)
				kwargs["additional_special_tokens"] += [
				    self.get_lang_token(a_	)
				    for lang_code in fairseq_language_code
				    if self.get_lang_token(a_	) not in kwargs["additional_special_tokens"]
				]

				super().__init__(
				    src_lang=a_						,			tgt_lang=a_						,			bos_token=a_						,			eos_token=a_						,			sep_token=a_						,			unk_token=a_						,			pad_token=a_						,			language_codes=a_						,			sp_model_kwargs=self.sp_model_kwargs						,			num_madeup_words=a_						,			**a_						,			)

				lowerCAmelCase_   :							int									=			vocab_file
				lowerCAmelCase_   :							Union[str, Any]									=			load_json(a_	)
				lowerCAmelCase_   :							str									=			{v: k for k, v in self.encoder.items()}
				lowerCAmelCase_   :							Dict									=			spm_file
				lowerCAmelCase_   :							Tuple									=			load_spm(a_						,			self.sp_model_kwargs	)

				lowerCAmelCase_   :							List[Any]									=			len(self.encoder	)

				lowerCAmelCase_   :							str									=			{
				    self.get_lang_token(a_	): self.encoder_size + i for i, lang_code in enumerate(a_	)
				}
				lowerCAmelCase_   :							Optional[Any]									=			{lang_code: self.encoder_size + i for i, lang_code in enumerate(a_	)}
				lowerCAmelCase_   :							str									=			{v: k for k, v in self.lang_token_to_id.items()}

				lowerCAmelCase_   :							Dict									=			src_lang if src_lang is not None else "en"
				lowerCAmelCase_   :							str									=			tgt_lang
				lowerCAmelCase_   :							List[Any]									=			self.get_lang_id(self._src_lang	)
				self.set_src_lang_special_tokens(self._src_lang	)

				lowerCAmelCase_   :							List[str]									=			num_madeup_words





		@property
		def    lowerCamelCase       (   self							:	List[str]	):
				return len(self.encoder	) + len(self.lang_token_to_id	)





		@property
		def    lowerCamelCase       (   self							:	List[str]	):
				return self._src_lang





		@src_lang.setter
		def    lowerCamelCase       (   self							:	Dict						,			a_							:	str	):
				lowerCAmelCase_   :							List[str]									=			new_src_lang
				self.set_src_lang_special_tokens(self._src_lang	)





		def    lowerCamelCase       (   self							:	Optional[Any]						,			a_							:	str	):
				return self.sp_model.encode(a_						,			out_type=a_	)





		def    lowerCamelCase       (   self							:	str						,			a_							:	List[Any]	):
				if token in self.lang_token_to_id:
						return self.lang_token_to_id[token]
				return self.encoder.get(a_						,			self.encoder[self.unk_token]	)





		def    lowerCamelCase       (   self							:	Tuple						,			a_							:	int	):
				if index in self.id_to_lang_token:
						return self.id_to_lang_token[index]
				return self.decoder.get(a_						,			self.unk_token	)





		def    lowerCamelCase       (   self							:	Optional[Any]						,			a_							:	Any	):
				lowerCAmelCase_   :							Optional[Any]									=			[]
				lowerCAmelCase_   :							int									=			""
				for token in tokens:
						# make sure that special tokens are not decoded using sentencepiece model
						if token in self.all_special_tokens:
								out_string += self.sp_model.decode(a_	) + token
								lowerCAmelCase_   :							Optional[Any]									=			[]
						else:
								current_sub_tokens.append(a_	)
				out_string += self.sp_model.decode(a_	)
				return out_string.strip()





		def    lowerCamelCase       (   self							:	Dict						,			a_							:	List[int]						,			a_							:	Optional[List[int]] = None						,			a_							:	bool = False	):

				if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=a_						,			token_ids_a=a_						,			already_has_special_tokens=a_	)

				lowerCAmelCase_   :							int									=			[1] * len(self.prefix_tokens	)
				lowerCAmelCase_   :							List[Any]									=			[1] * len(self.suffix_tokens	)
				if token_ids_a is None:
						return prefix_ones + ([0] * len(a_	)) + suffix_ones
				return prefix_ones + ([0] * len(a_	)) + ([0] * len(a_	)) + suffix_ones





		def    lowerCamelCase       (   self							:	Tuple						,			a_							:	List[int]						,			a_							:	Optional[List[int]] = None	):
				if token_ids_a is None:
						return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





		def    lowerCamelCase       (   self							:	str	):
				lowerCAmelCase_   :							int									=			{self.convert_ids_to_tokens(a_	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab





		def __getstate__(   self							:	List[Any]	):
				lowerCAmelCase_   :							Union[str, Any]									=			self.__dict__.copy()
				lowerCAmelCase_   :							Tuple									=			None
				return state





		def __setstate__(   self							:	Optional[Any]						,			a_							:	Dict	):
				lowerCAmelCase_   :							List[Any]									=			d

				# for backward compatibility
				if not hasattr(self						,			"sp_model_kwargs"	):
						lowerCAmelCase_   :							Union[str, Any]									=			{}

				lowerCAmelCase_   :							Tuple									=			load_spm(self.spm_file						,			self.sp_model_kwargs	)





		def    lowerCamelCase       (   self							:	Dict						,			a_							:	str						,			a_							:	Optional[str] = None	):
				lowerCAmelCase_   :							Union[str, Any]									=			Path(a_	)
				if not save_dir.is_dir():
						raise OSError(f'''{save_directory} should be a directory'''	)
				lowerCAmelCase_   :							str									=			save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
				)
				lowerCAmelCase_   :							Dict									=			save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
				)

				save_json(self.encoder						,			a_	)

				if os.path.abspath(self.spm_file	) != os.path.abspath(a_	) and os.path.isfile(self.spm_file	):
						copyfile(self.spm_file						,			a_	)
				elif not os.path.isfile(self.spm_file	):
						with open(a_						,			"wb"	) as fi:
								lowerCAmelCase_   :							Any									=			self.sp_model.serialized_model_proto()
								fi.write(a_	)

				return (str(a_	), str(a_	))





		def    lowerCamelCase       (   self							:	str						,			a_							:	List[str]						,			a_							:	str = "en"						,			a_							:	Optional[List[str]] = None						,			a_							:	str = "ro"						,			**a_							:	Union[str, Any]						,			):
				lowerCAmelCase_   :							Optional[int]									=			src_lang
				lowerCAmelCase_   :							List[str]									=			tgt_lang
				self.set_src_lang_special_tokens(self.src_lang	)
				return super().prepare_seqaseq_batch(a_						,			a_						,			**a_	)





		def    lowerCamelCase       (   self							:	str						,			a_							:	List[str]						,			a_							:	Optional[str]						,			a_							:	Optional[str]						,			**a_							:	List[Any]	):
				if src_lang is None or tgt_lang is None:
						raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"	)
				lowerCAmelCase_   :							Tuple									=			src_lang
				lowerCAmelCase_   :							Dict									=			self(a_						,			add_special_tokens=a_						,			**a_	)
				lowerCAmelCase_   :							str									=			self.get_lang_id(a_	)
				lowerCAmelCase_   :							Optional[Any]									=			tgt_lang_id
				return inputs





		def    lowerCamelCase       (   self							:	Optional[Any]	):
				self.set_src_lang_special_tokens(self.src_lang	)





		def    lowerCamelCase       (   self							:	Optional[int]	):
				self.set_tgt_lang_special_tokens(self.tgt_lang	)





		def    lowerCamelCase       (   self							:	int						,			a_							:	str	):
				lowerCAmelCase_   :							Dict									=			self.get_lang_token(a_	)
				lowerCAmelCase_   :							Union[str, Any]									=			self.lang_token_to_id[lang_token]
				lowerCAmelCase_   :							List[Any]									=			[self.cur_lang_id]
				lowerCAmelCase_   :							List[Any]									=			[self.eos_token_id]





		def    lowerCamelCase       (   self							:	Dict						,			a_							:	str	):
				lowerCAmelCase_   :							List[Any]									=			self.get_lang_token(a_	)
				lowerCAmelCase_   :							Tuple									=			self.lang_token_to_id[lang_token]
				lowerCAmelCase_   :							str									=			[self.cur_lang_id]
				lowerCAmelCase_   :							int									=			[self.eos_token_id]





		def    lowerCamelCase       (   self							:	Optional[Any]						,			a_							:	str	):
				return self.lang_code_to_token[lang]





		def    lowerCamelCase       (   self							:	Any						,			a_							:	str	):
				lowerCAmelCase_   :							Dict									=			self.get_lang_token(a_	)
				return self.lang_token_to_id[lang_token]







def 		__lowerCamelCase	(  __UpperCamelCase      ,       __UpperCamelCase       )						->    Optional[Any]:

		"""simple docstring"""







		lowerCAmelCase_   :							Dict									=			sentencepiece.SentencePieceProcessor(**__UpperCamelCase       )
		spm.Load(str(__UpperCamelCase       )       )
		return spm

def 		__lowerCamelCase	(  __UpperCamelCase       )						->    List[Any]:

		"""simple docstring"""







		with open(__UpperCamelCase      ,       "r"       ) as f:
				return json.load(__UpperCamelCase       )



def 		__lowerCamelCase	(  __UpperCamelCase      ,       __UpperCamelCase       )						->    Union[str, Any]:

		"""simple docstring"""







		with open(__UpperCamelCase      ,       "w"       ) as f:
				json.dump(__UpperCamelCase      ,       __UpperCamelCase      ,       indent=2       )

import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
A   :     Tuple       =				"src/transformers"
A   :     Optional[Any]       =				"docs/source/en/tasks"


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase   ,					__UpperCamelCase ):
      with open(__UpperCamelCase   ,					"r"   ,					encoding="utf-8"   ,					newline="\n" ) as f:
            SCREAMING_SNAKE_CASE_      =     f.readlines()
      # Find the start prompt.
      SCREAMING_SNAKE_CASE_      =     0
      while not lines[start_index].startswith(__UpperCamelCase ):
            start_index += 1
      start_index += 1

      SCREAMING_SNAKE_CASE_      =     start_index
      while not lines[end_index].startswith(__UpperCamelCase ):
            end_index += 1
      end_index -= 1

      while len(lines[start_index] ) <= 1:
            start_index += 1
      while len(lines[end_index] ) <= 1:
            end_index -= 1
      end_index += 1
      return "".join(lines[start_index:end_index] ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
A   :     List[str]       =				direct_transformers_import(TRANSFORMERS_PATH)

A   :     List[Any]       =				{
    "asr.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    "audio_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    "language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    "image_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    "masked_language_modeling.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    "multiple_choice.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    "object_detection.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    "question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    "semantic_segmentation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    "sequence_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    "summarization.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "token_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    "translation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    "video_classification.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    "document_question_answering.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    "monocular_depth_estimation.md": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
A   :     Any       =				{
    "summarization.md": ("nllb",),
    "translation.md": ("nllb",),
}


def    a__       (						__UpperCamelCase ):
      SCREAMING_SNAKE_CASE_      =     TASK_GUIDE_TO_MODELS[task_guide]
      SCREAMING_SNAKE_CASE_      =     SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(__UpperCamelCase   ,					set() )
      SCREAMING_SNAKE_CASE_      =     {
          code: name
          for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
          if (code in model_maping_names or code in special_model_types)
      }
      return ", ".join([F'''[{name}](../model_doc/{code})''' for code, name in model_names.items()] ) + "\n"


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase=False ):
      SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_      =     _find_text_in_file(
          filename=os.path.join(__UpperCamelCase   ,					__UpperCamelCase )   ,					start_prompt="<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->"   ,					end_prompt="<!--End of the generated tip-->"   ,					)

      SCREAMING_SNAKE_CASE_      =     get_model_list_for_task(__UpperCamelCase )

      if current_list != new_list:
            if overwrite:
                  with open(os.path.join(__UpperCamelCase   ,					__UpperCamelCase )   ,					"w"   ,					encoding="utf-8"   ,					newline="\n" ) as f:
                        f.writelines(lines[:start_index] + [new_list] + lines[end_index:] )
            else:
                  raise ValueError(
                      F'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
                      " to fix this." )


if __name__ == "__main__":
      A   :     Tuple       =				argparse.ArgumentParser()
      parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
      A   :     Dict       =				parser.parse_args()

      for task_guide in TASK_GUIDE_TO_MODELS.keys():
            check_model_list_for_task(task_guide, args.fix_and_overwrite)


