





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_						   =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_						   =       """▁"""

SCREAMING_SNAKE_CASE_						   =       {"""vocab_file""": """sentencepiece.bpe.model"""}

SCREAMING_SNAKE_CASE_						   =       {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

SCREAMING_SNAKE_CASE_						   =       {
    """facebook/mbart-large-en-ro""": 1_0_2_4,
    """facebook/mbart-large-cc25""": 1_0_2_4,
}

# fmt: off
SCREAMING_SNAKE_CASE_						   =       ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]

class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     __snake_case    :	List[Any]			=			VOCAB_FILES_NAMES
     __snake_case    :	List[Any]			=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case    :	Any			=			PRETRAINED_VOCAB_FILES_MAP
     __snake_case    :	Optional[Any]			=			["input_ids", "attention_mask"]

     __snake_case    :	List[int]			=			[]
     __snake_case    :	List[int]			=			[]






     def __init__(							self						:    Any						,lowerCamelCase__						:    Dict						,lowerCamelCase__						:    Any="<s>"						,lowerCamelCase__						:    Optional[int]="</s>"						,lowerCamelCase__						:    Optional[Any]="</s>"						,lowerCamelCase__						:    Optional[Any]="<s>"						,lowerCamelCase__						:    Dict="<unk>"						,lowerCamelCase__						:    List[str]="<pad>"						,lowerCamelCase__						:    Any="<mask>"						,lowerCamelCase__						:    List[Any]=None						,lowerCamelCase__						:    int=None						,lowerCamelCase__						:    Tuple=None						,lowerCamelCase__						:    Optional[Dict[str, Any]] = None						,lowerCamelCase__						:    str=None						,**lowerCamelCase__						:    str						,)     ->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						AddedToken(lowerCamelCase__						,lstrip=lowerCamelCase__						,rstrip=lowerCamelCase__       ) if isinstance(lowerCamelCase__						,lowerCamelCase__       ) else mask_token

           SCREAMING_SNAKE_CASE		=						{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               bos_token=lowerCamelCase__						,eos_token=lowerCamelCase__						,unk_token=lowerCamelCase__						,sep_token=lowerCamelCase__						,cls_token=lowerCamelCase__						,pad_token=lowerCamelCase__						,mask_token=lowerCamelCase__						,tokenizer_file=lowerCamelCase__						,src_lang=lowerCamelCase__						,tgt_lang=lowerCamelCase__						,additional_special_tokens=lowerCamelCase__						,sp_model_kwargs=self.sp_model_kwargs						,**lowerCamelCase__						,)

           SCREAMING_SNAKE_CASE		=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
           self.sp_model.Load(str(lowerCamelCase__       )       )
           SCREAMING_SNAKE_CASE		=						vocab_file

           # Original fairseq vocab and spm vocab must be "aligned":
           # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
           # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
           # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
           # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

           # Mimic fairseq token-to-id alignment for the first 4 token
           SCREAMING_SNAKE_CASE		=						{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

           # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
           SCREAMING_SNAKE_CASE		=						1

           SCREAMING_SNAKE_CASE		=						len(self.sp_model       )
           SCREAMING_SNAKE_CASE		=						{
               code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(lowerCamelCase__       )
           }
           SCREAMING_SNAKE_CASE		=						{v: k for k, v in self.lang_code_to_id.items()}
           SCREAMING_SNAKE_CASE		=						len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

           self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
           SCREAMING_SNAKE_CASE		=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
           SCREAMING_SNAKE_CASE		=						list(self.lang_code_to_id.keys()       )

           if additional_special_tokens is not None:
                 # Only add those special tokens if they are not already there.
                 self._additional_special_tokens.extend(
                     [t for t in additional_special_tokens if t not in self._additional_special_tokens]       )

           SCREAMING_SNAKE_CASE		=						src_lang if src_lang is not None else """en_XX"""
           SCREAMING_SNAKE_CASE		=						self.lang_code_to_id[self._src_lang]
           SCREAMING_SNAKE_CASE		=						tgt_lang
           self.set_src_lang_special_tokens(self._src_lang       )






     def __getstate__(							self						:    List[Any]       )     ->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.__dict__.copy()
           SCREAMING_SNAKE_CASE		=						None
           SCREAMING_SNAKE_CASE		=						self.sp_model.serialized_model_proto()
           return state






     def __setstate__(							self						:    Union[str, Any]						,lowerCamelCase__						:    Any       )     ->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						d

           # for backward compatibility
           if not hasattr(self						,"""sp_model_kwargs"""       ):
                 SCREAMING_SNAKE_CASE		=						{}

           SCREAMING_SNAKE_CASE		=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
           self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )






     @property
     def 				SCREAMING_SNAKE_CASE__  (							self						:    int       )     ->		Union[str, Any]:




           '''simple docstring'''


           return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token






     @property
     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		str:




           '''simple docstring'''


           return self._src_lang






     @src_lang.setter
     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[Any]						,lowerCamelCase__						:    str       )     ->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						new_src_lang
           self.set_src_lang_special_tokens(self._src_lang       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]						,lowerCamelCase__						:    List[int]						,lowerCamelCase__						:    Optional[List[int]] = None						,lowerCamelCase__						:    bool = False       )     ->		List[int]:




           '''simple docstring'''


           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=lowerCamelCase__						,token_ids_a=lowerCamelCase__						,already_has_special_tokens=lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						[1] * len(self.prefix_tokens       )
           SCREAMING_SNAKE_CASE		=						[1] * len(self.suffix_tokens       )
           if token_ids_a is None:
                 return prefix_ones + ([0] * len(lowerCamelCase__       )) + suffix_ones
           return prefix_ones + ([0] * len(lowerCamelCase__       )) + ([0] * len(lowerCamelCase__       )) + suffix_ones






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Any						,lowerCamelCase__						:    List[int]						,lowerCamelCase__						:    Optional[List[int]] = None       )     ->		List[int]:




           '''simple docstring'''


           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]						,lowerCamelCase__						:    List[int]						,lowerCamelCase__						:    Optional[List[int]] = None       )     ->		List[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						[self.sep_token_id]
           SCREAMING_SNAKE_CASE		=						[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep       ) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]						,lowerCamelCase__						:    Dict						,lowerCamelCase__						:    str						,lowerCamelCase__						:    Optional[str]						,lowerCamelCase__						:    Optional[str]						,**lowerCamelCase__						:    Union[str, Any]       )     ->		List[str]:




           '''simple docstring'''


           if src_lang is None or tgt_lang is None:
                 raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""       )
           SCREAMING_SNAKE_CASE		=						src_lang
           SCREAMING_SNAKE_CASE		=						self(lowerCamelCase__						,add_special_tokens=lowerCamelCase__						,return_tensors=lowerCamelCase__						,**lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						self.convert_tokens_to_ids(lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						tgt_lang_id
           return inputs






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[str]       )     ->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						{self.convert_ids_to_tokens(lowerCamelCase__       ): i for i in range(self.vocab_size       )}
           vocab.update(self.added_tokens_encoder       )
           return vocab






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple						,lowerCamelCase__						:    str       )     ->		List[str]:




           '''simple docstring'''


           return self.sp_model.encode(lowerCamelCase__						,out_type=lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Union[str, Any]						,lowerCamelCase__						:    Tuple       )     ->		Tuple:




           '''simple docstring'''


           if token in self.fairseq_tokens_to_ids:
                 return self.fairseq_tokens_to_ids[token]
           SCREAMING_SNAKE_CASE		=						self.sp_model.PieceToId(lowerCamelCase__       )

           # Need to return unknown token if the SP model returned 0
           return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






     def 				SCREAMING_SNAKE_CASE__  (							self						:    List[Any]						,lowerCamelCase__						:    Optional[Any]       )     ->		str:




           '''simple docstring'''


           if index in self.fairseq_ids_to_tokens:
                 return self.fairseq_ids_to_tokens[index]
           return self.sp_model.IdToPiece(index - self.fairseq_offset       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int						,lowerCamelCase__						:    Dict       )     ->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""""".join(lowerCamelCase__       ).replace(lowerCamelCase__						,""" """       ).strip()
           return out_string






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int						,lowerCamelCase__						:    str						,lowerCamelCase__						:    Optional[str] = None       )     ->		Tuple[str]:




           '''simple docstring'''


           if not os.path.isdir(lowerCamelCase__       ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
                 return
           SCREAMING_SNAKE_CASE		=						os.path.join(
               lowerCamelCase__						,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

           if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCamelCase__       ) and os.path.isfile(self.vocab_file       ):
                 copyfile(self.vocab_file						,lowerCamelCase__       )
           elif not os.path.isfile(self.vocab_file       ):
                 with open(lowerCamelCase__						,"""wb"""       ) as fi:
                       SCREAMING_SNAKE_CASE		=						self.sp_model.serialized_model_proto()
                       fi.write(lowerCamelCase__       )

           return (out_vocab_file,)






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[int]						,lowerCamelCase__						:    List[str]						,lowerCamelCase__						:    str = "en_XX"						,lowerCamelCase__						:    Optional[List[str]] = None						,lowerCamelCase__						:    str = "ro_RO"						,**lowerCamelCase__						:    List[str]						,)     ->		BatchEncoding:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						src_lang
           SCREAMING_SNAKE_CASE		=						tgt_lang
           return super().prepare_seqaseq_batch(lowerCamelCase__						,lowerCamelCase__						,**lowerCamelCase__       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		Optional[int]:




           '''simple docstring'''


           return self.set_src_lang_special_tokens(self.src_lang       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Optional[Any]       )     ->		int:




           '''simple docstring'''


           return self.set_tgt_lang_special_tokens(self.tgt_lang       )






     def 				SCREAMING_SNAKE_CASE__  (							self						:    int						,lowerCamelCase__						:    Union[str, Any]       )     ->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.lang_code_to_id[src_lang]
           SCREAMING_SNAKE_CASE		=						[]
           SCREAMING_SNAKE_CASE		=						[self.eos_token_id, self.cur_lang_code]






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Tuple						,lowerCamelCase__						:    str       )     ->		None:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.lang_code_to_id[lang]
           SCREAMING_SNAKE_CASE		=						[]
           SCREAMING_SNAKE_CASE		=						[self.eos_token_id, self.cur_lang_code]









import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_						   =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_						   =       {
    """microsoft/git-base""": """https://huggingface.co/microsoft/git-base/resolve/main/config.json""",
}

class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     __snake_case    :	Dict			=			"git_vision_model"






     def __init__(							self						:    List[Any]						,lowerCamelCase__						:    Dict=768						,lowerCamelCase__						:    Union[str, Any]=3072						,lowerCamelCase__						:    Optional[int]=12						,lowerCamelCase__						:    Tuple=12						,lowerCamelCase__						:    Tuple=3						,lowerCamelCase__						:    Optional[Any]=224						,lowerCamelCase__						:    Union[str, Any]=16						,lowerCamelCase__						:    List[Any]="quick_gelu"						,lowerCamelCase__						:    Optional[Any]=1e-5						,lowerCamelCase__						:    str=0.0						,lowerCamelCase__						:    Optional[int]=0.02						,**lowerCamelCase__						:    Union[str, Any]						,)     ->		Optional[int]:




           '''simple docstring'''


           super().__init__(**lowerCamelCase__       )

           SCREAMING_SNAKE_CASE		=						hidden_size
           SCREAMING_SNAKE_CASE		=						intermediate_size
           SCREAMING_SNAKE_CASE		=						num_hidden_layers
           SCREAMING_SNAKE_CASE		=						num_attention_heads
           SCREAMING_SNAKE_CASE		=						num_channels
           SCREAMING_SNAKE_CASE		=						patch_size
           SCREAMING_SNAKE_CASE		=						image_size
           SCREAMING_SNAKE_CASE		=						initializer_range
           SCREAMING_SNAKE_CASE		=						attention_dropout
           SCREAMING_SNAKE_CASE		=						layer_norm_eps
           SCREAMING_SNAKE_CASE		=						hidden_act






     @classmethod
     def 				SCREAMING_SNAKE_CASE__  (							cls						:    Tuple						,lowerCamelCase__						:    Union[str, os.PathLike]						,**lowerCamelCase__						:    int       )     ->		"PretrainedConfig":




           '''simple docstring'''


           cls._set_token_in_kwargs(lowerCamelCase__       )

           SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE		=						cls.get_config_dict(lowerCamelCase__						,**lowerCamelCase__       )

           # get the vision config dict if we are loading from GITConfig
           if config_dict.get("""model_type"""       ) == "git":
                 SCREAMING_SNAKE_CASE		=						config_dict["""vision_config"""]

           if "model_type" in config_dict and hasattr(cls						,"""model_type"""       ) and config_dict["model_type"] != cls.model_type:
                 logger.warning(
                     F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
                     F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""       )

           return cls.from_dict(lowerCamelCase__						,**lowerCamelCase__       )

class 		UpperCamelCase__ (      lowerCAmelCase_   ):



     '''simple docstring'''

     __snake_case    :	Dict			=			"git"






     def __init__(							self						:    Optional[int]						,lowerCamelCase__						:    int=None						,lowerCamelCase__						:    str=30522						,lowerCamelCase__						:    Tuple=768						,lowerCamelCase__						:    Union[str, Any]=6						,lowerCamelCase__						:    str=12						,lowerCamelCase__						:    List[str]=3072						,lowerCamelCase__						:    Dict="gelu"						,lowerCamelCase__						:    Tuple=0.1						,lowerCamelCase__						:    Any=0.1						,lowerCamelCase__						:    List[str]=1024						,lowerCamelCase__						:    List[str]=0.02						,lowerCamelCase__						:    str=1e-1_2						,lowerCamelCase__						:    Optional[int]=0						,lowerCamelCase__						:    Optional[int]="absolute"						,lowerCamelCase__						:    Optional[Any]=True						,lowerCamelCase__						:    str=False						,lowerCamelCase__						:    int=101						,lowerCamelCase__						:    int=102						,lowerCamelCase__						:    Dict=None						,**lowerCamelCase__						:    List[Any]						,)     ->		Optional[Any]:




           '''simple docstring'''


           super().__init__(bos_token_id=lowerCamelCase__						,eos_token_id=lowerCamelCase__						,pad_token_id=lowerCamelCase__						,**lowerCamelCase__       )

           if vision_config is None:
                 SCREAMING_SNAKE_CASE		=						{}
                 logger.info("""vision_config is None. initializing the GitVisionConfig with default values."""       )

           SCREAMING_SNAKE_CASE		=						GitVisionConfig(**lowerCamelCase__       )
           SCREAMING_SNAKE_CASE		=						vocab_size
           SCREAMING_SNAKE_CASE		=						hidden_size
           SCREAMING_SNAKE_CASE		=						num_hidden_layers
           SCREAMING_SNAKE_CASE		=						num_attention_heads
           SCREAMING_SNAKE_CASE		=						hidden_act
           SCREAMING_SNAKE_CASE		=						intermediate_size
           SCREAMING_SNAKE_CASE		=						hidden_dropout_prob
           SCREAMING_SNAKE_CASE		=						attention_probs_dropout_prob
           SCREAMING_SNAKE_CASE		=						max_position_embeddings
           SCREAMING_SNAKE_CASE		=						initializer_range
           SCREAMING_SNAKE_CASE		=						layer_norm_eps
           SCREAMING_SNAKE_CASE		=						position_embedding_type
           SCREAMING_SNAKE_CASE		=						use_cache
           SCREAMING_SNAKE_CASE		=						tie_word_embeddings
           SCREAMING_SNAKE_CASE		=						num_image_with_embedding

           SCREAMING_SNAKE_CASE		=						bos_token_id
           SCREAMING_SNAKE_CASE		=						eos_token_id






     def 				SCREAMING_SNAKE_CASE__  (							self						:    Dict       )     ->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						copy.deepcopy(self.__dict__       )
           SCREAMING_SNAKE_CASE		=						self.vision_config.to_dict()
           SCREAMING_SNAKE_CASE		=						self.__class__.model_type
           return output


