


'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__snake_case							      =     logging.get_logger(__name__)

__snake_case							      =     '''▁'''

__snake_case							      =     {'''vocab_file''': '''sentencepiece.bpe.model'''}

__snake_case							      =     {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

__snake_case							      =     {
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
__snake_case							      =     ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']
class    lowercase	(							A__					):

		"""simple docstring"""

		_a     =				VOCAB_FILES_NAMES
		_a     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     =				PRETRAINED_VOCAB_FILES_MAP
		_a     =				['input_ids', 'attention_mask']

		_a     =				[]
		_a     =				[]




		def __init__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_="<s>"    ,   UpperCamelCase_="</s>"    ,   UpperCamelCase_="</s>"    ,   UpperCamelCase_="<s>"    ,   UpperCamelCase_="<unk>"    ,   UpperCamelCase_="<pad>"    ,   UpperCamelCase_="<mask>"    ,   UpperCamelCase_=None    ,   UpperCamelCase_=None    ,   UpperCamelCase_=None    ,   UpperCamelCase_ = None    ,   UpperCamelCase_=None    ,   **UpperCamelCase_    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :Dict												= AddedToken(UpperCamelCase_    ,   lstrip=UpperCamelCase_    ,   rstrip=UpperCamelCase_							) if isinstance(UpperCamelCase_    ,   UpperCamelCase_							) else mask_token

							UpperCamelCase__   :int												= {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=UpperCamelCase_    ,   eos_token=UpperCamelCase_    ,   unk_token=UpperCamelCase_    ,   sep_token=UpperCamelCase_    ,   cls_token=UpperCamelCase_    ,   pad_token=UpperCamelCase_    ,   mask_token=UpperCamelCase_    ,   tokenizer_file=UpperCamelCase_    ,   src_lang=UpperCamelCase_    ,   tgt_lang=UpperCamelCase_    ,   additional_special_tokens=UpperCamelCase_    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **UpperCamelCase_    ,   )

							UpperCamelCase__   :int												= spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(str(UpperCamelCase_							)							)
							UpperCamelCase__   :Optional[int]												= vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCamelCase__   :Dict												= {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCamelCase__   :Tuple												= 1

							UpperCamelCase__   :int												= len(self.sp_model							)
							UpperCamelCase__   :Dict												= {
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_							)
							}
							UpperCamelCase__   :List[Any]												= {v: k for k, v in self.lang_code_to_id.items()}
							UpperCamelCase__   :Any												= len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id							)
							UpperCamelCase__   :Union[str, Any]												= {v: k for k, v in self.fairseq_tokens_to_ids.items()}
							UpperCamelCase__   :Union[str, Any]												= list(self.lang_code_to_id.keys()							)

							if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												self._additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in self._additional_special_tokens]							)

							UpperCamelCase__   :Any												= src_lang if src_lang is not None else '''en_XX'''
							UpperCamelCase__   :Optional[Any]												= self.lang_code_to_id[self._src_lang]
							UpperCamelCase__   :Union[str, Any]												= tgt_lang
							self.set_src_lang_special_tokens(self._src_lang							)




		def __getstate__(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Dict												= self.__dict__.copy()
							UpperCamelCase__   :int												= None
							UpperCamelCase__   :Dict												= self.sp_model.serialized_model_proto()
							return state




		def __setstate__(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= d

							# for backward compatibility
							if not hasattr(self    ,   '''sp_model_kwargs'''							):
												UpperCamelCase__   :Optional[int]												= {}

							UpperCamelCase__   :Optional[Any]												= spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)




		@property
		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							return len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset + 1  # Plus 1 for the mask token




		@property
		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							return self._src_lang




		@src_lang.setter
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Optional[Any]												= new_src_lang
							self.set_src_lang_special_tokens(self._src_lang							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = False							):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_    ,   token_ids_a=UpperCamelCase_    ,   already_has_special_tokens=UpperCamelCase_							)

							UpperCamelCase__   :List[str]												= [1] * len(self.prefix_tokens							)
							UpperCamelCase__   :int												= [1] * len(self.suffix_tokens							)
							if token_ids_a is None:
												return prefix_ones + ([0] * len(UpperCamelCase_							)) + suffix_ones
							return prefix_ones + ([0] * len(UpperCamelCase_							)) + ([0] * len(UpperCamelCase_							)) + suffix_ones




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = None							):

							'''simple docstring'''


							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__   :Optional[int]												= [self.sep_token_id]
							UpperCamelCase__   :List[Any]												= [self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   **UpperCamelCase_							):

							'''simple docstring'''


							if src_lang is None or tgt_lang is None:
												raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''							)
							UpperCamelCase__   :Tuple												= src_lang
							UpperCamelCase__   :Optional[Any]												= self(UpperCamelCase_    ,   add_special_tokens=UpperCamelCase_    ,   return_tensors=UpperCamelCase_    ,   **UpperCamelCase_							)
							UpperCamelCase__   :List[str]												= self.convert_tokens_to_ids(UpperCamelCase_							)
							UpperCamelCase__   :Dict												= tgt_lang_id
							return inputs




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= {self.convert_ids_to_tokens(UpperCamelCase_							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							return self.sp_model.encode(UpperCamelCase_    ,   out_type=UpperCamelCase_							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCamelCase__   :Any												= self.sp_model.PieceToId(UpperCamelCase_							)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :List[str]												= ''''''.join(UpperCamelCase_							).replace(UpperCamelCase_    ,   ''' '''							).strip()
							return out_string




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = None							):

							'''simple docstring'''


							if not os.path.isdir(UpperCamelCase_							):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
												return
							UpperCamelCase__   :int												= os.path.join(
							    UpperCamelCase_    ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCamelCase_							) and os.path.isfile(self.vocab_file							):
												copyfile(self.vocab_file    ,   UpperCamelCase_							)
							elif not os.path.isfile(self.vocab_file							):
												with open(UpperCamelCase_    ,   '''wb'''							) as fi:
																	UpperCamelCase__   :Any												= self.sp_model.serialized_model_proto()
																	fi.write(UpperCamelCase_							)

							return (out_vocab_file,)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_ = "en_XX"    ,   UpperCamelCase_ = None    ,   UpperCamelCase_ = "ro_RO"    ,   **UpperCamelCase_    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :Optional[Any]												= src_lang
							UpperCamelCase__   :Optional[Any]												= tgt_lang
							return super().prepare_seqaseq_batch(UpperCamelCase_    ,   UpperCamelCase_    ,   **UpperCamelCase_							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							return self.set_src_lang_special_tokens(self.src_lang							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							return self.set_tgt_lang_special_tokens(self.tgt_lang							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Any												= self.lang_code_to_id[src_lang]
							UpperCamelCase__   :int												= []
							UpperCamelCase__   :Union[str, Any]												= [self.eos_token_id, self.cur_lang_code]




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :Dict												= self.lang_code_to_id[lang]
							UpperCamelCase__   :Optional[Any]												= []
							UpperCamelCase__   :Tuple												= [self.eos_token_id, self.cur_lang_code]


'''simple docstring'''



class 						lowerCAmelCase__   :



 """simple docstring"""



 def __init__(		self					:       List[Any]					,							__SCREAMING_SNAKE_CASE					:       int			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      n
      __SCREAMING_SNAKE_CASE     =      [None] * self.n
      __SCREAMING_SNAKE_CASE     =      0  # index of the first element
      __SCREAMING_SNAKE_CASE     =      0
      __SCREAMING_SNAKE_CASE     =      0
 def __len__(		self					:       Tuple			)  -> int:


      """simple docstring"""




      return self.size
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> bool:


      """simple docstring"""




      return self.size == 0
 def       UpperCAmelCase__		(		self					:       Any			)  -> int:


      """simple docstring"""




      return False if self.is_empty() else self.array[self.front]
 def       UpperCAmelCase__		(		self					:       Dict					,							__SCREAMING_SNAKE_CASE					:       Union[str, Any]			)  -> Dict:


      """simple docstring"""




      if self.size >= self.n:
           raise Exception("""QUEUE IS FULL"""			)

      __SCREAMING_SNAKE_CASE     =      data
      __SCREAMING_SNAKE_CASE     =      (self.rear + 1) % self.n
      self.size += 1
      return self


 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Optional[Any]:


      """simple docstring"""




      if self.size == 0:
           raise Exception("""UNDERFLOW"""			)

      __SCREAMING_SNAKE_CASE     =      self.array[self.front]
      __SCREAMING_SNAKE_CASE     =      None
      __SCREAMING_SNAKE_CASE     =      (self.front + 1) % self.n
      self.size -= 1
      return temp






