



'''simple docstring'''

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase									=   logging.get_logger(__name__)

__lowerCAmelCase									=   '''▁'''

__lowerCAmelCase									=   {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__lowerCAmelCase									=   {
    '''vocab_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json''',
    },
    '''spm_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_config_file''': {
        '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json''',
        '''facebook/m2m100_1.2B''': '''https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json''',
    },
}

__lowerCAmelCase									=   {
    '''facebook/m2m100_418M''': 1_024,
}

# fmt: off
__lowerCAmelCase									=   {
    '''m2m100''': ['''af''', '''am''', '''ar''', '''ast''', '''az''', '''ba''', '''be''', '''bg''', '''bn''', '''br''', '''bs''', '''ca''', '''ceb''', '''cs''', '''cy''', '''da''', '''de''', '''el''', '''en''', '''es''', '''et''', '''fa''', '''ff''', '''fi''', '''fr''', '''fy''', '''ga''', '''gd''', '''gl''', '''gu''', '''ha''', '''he''', '''hi''', '''hr''', '''ht''', '''hu''', '''hy''', '''id''', '''ig''', '''ilo''', '''is''', '''it''', '''ja''', '''jv''', '''ka''', '''kk''', '''km''', '''kn''', '''ko''', '''lb''', '''lg''', '''ln''', '''lo''', '''lt''', '''lv''', '''mg''', '''mk''', '''ml''', '''mn''', '''mr''', '''ms''', '''my''', '''ne''', '''nl''', '''no''', '''ns''', '''oc''', '''or''', '''pa''', '''pl''', '''ps''', '''pt''', '''ro''', '''ru''', '''sd''', '''si''', '''sk''', '''sl''', '''so''', '''sq''', '''sr''', '''ss''', '''su''', '''sv''', '''sw''', '''ta''', '''th''', '''tl''', '''tn''', '''tr''', '''uk''', '''ur''', '''uz''', '''vi''', '''wo''', '''xh''', '''yi''', '''yo''', '''zh''', '''zu'''],
    '''wmt21''': ['''en''', '''ha''', '''is''', '''ja''', '''cs''', '''ru''', '''zh''', '''de''']
}

class 	__magic_name__						(    _UpperCamelCase   ):
							lowerCAmelCase       :    List[Any]   				=					VOCAB_FILES_NAMES
							lowerCAmelCase       :    List[str]   				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase       :    List[str]   				=					PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase       :    Union[str, Any]   				=					['input_ids', 'attention_mask']

							lowerCAmelCase       :    List[int]   				=					[]
							lowerCAmelCase       :    List[int]   				=					[]







							def __init__(							self					:       List[Any]	,_UpperCAmelCase					:       str	,_UpperCAmelCase					:       Any	,_UpperCAmelCase					:       Union[str, Any]=None	,_UpperCAmelCase					:       Optional[Any]=None	,_UpperCAmelCase					:       Union[str, Any]="<s>"	,_UpperCAmelCase					:       Union[str, Any]="</s>"	,_UpperCAmelCase					:       int="</s>"	,_UpperCAmelCase					:       Tuple="<pad>"	,_UpperCAmelCase					:       str="<unk>"	,_UpperCAmelCase					:       Tuple="m2m100"	,_UpperCAmelCase					:       Optional[Dict[str, Any]] = None	,_UpperCAmelCase					:       List[Any]=8	,**_UpperCAmelCase					:       Union[str, Any]	,):
									_a     :      int        =							{} if sp_model_kwargs is None else sp_model_kwargs

									_a     :      int        =							language_codes
									_a     :      List[Any]        =							FAIRSEQ_LANGUAGE_CODES[language_codes]
									_a     :      Union[str, Any]        =							{lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

									_a     :      List[str]        =							kwargs.get('additional_special_tokens'	,[]		)
									kwargs["additional_special_tokens"] += [
									    self.get_lang_token(_UpperCAmelCase		)
									    for lang_code in fairseq_language_code
									    if self.get_lang_token(_UpperCAmelCase		) not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    src_lang=_UpperCAmelCase	,tgt_lang=_UpperCAmelCase	,bos_token=_UpperCAmelCase	,eos_token=_UpperCAmelCase	,sep_token=_UpperCAmelCase	,unk_token=_UpperCAmelCase	,pad_token=_UpperCAmelCase	,language_codes=_UpperCAmelCase	,sp_model_kwargs=self.sp_model_kwargs	,num_madeup_words=_UpperCAmelCase	,**_UpperCAmelCase	,)

									_a     :      Any        =							vocab_file
									_a     :      int        =							load_json(_UpperCAmelCase		)
									_a     :      Dict        =							{v: k for k, v in self.encoder.items()}
									_a     :      int        =							spm_file
									_a     :      int        =							load_spm(_UpperCAmelCase	,self.sp_model_kwargs		)

									_a     :      Any        =							len(self.encoder		)

									_a     :      str        =							{
									    self.get_lang_token(_UpperCAmelCase		): self.encoder_size + i for i, lang_code in enumerate(_UpperCAmelCase		)
									}
									_a     :      Optional[int]        =							{lang_code: self.encoder_size + i for i, lang_code in enumerate(_UpperCAmelCase		)}
									_a     :      Dict        =							{v: k for k, v in self.lang_token_to_id.items()}

									_a     :      Union[str, Any]        =							src_lang if src_lang is not None else 'en'
									_a     :      Optional[int]        =							tgt_lang
									_a     :      Optional[Any]        =							self.get_lang_id(self._src_lang		)
									self.set_src_lang_special_tokens(self._src_lang		)

									_a     :      List[Any]        =							num_madeup_words







							@property
							def 							__lowercase			(							self					:       int		):
									return len(self.encoder		) + len(self.lang_token_to_id		)







							@property
							def 							__lowercase			(							self					:       Optional[int]		):
									return self._src_lang







							@src_lang.setter
							def 							__lowercase			(							self					:       str	,_UpperCAmelCase					:       str		):
									_a     :      Dict        =							new_src_lang
									self.set_src_lang_special_tokens(self._src_lang		)







							def 							__lowercase			(							self					:       Union[str, Any]	,_UpperCAmelCase					:       str		):
									return self.sp_model.encode(_UpperCAmelCase	,out_type=_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       int		):
									if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
									return self.encoder.get(_UpperCAmelCase	,self.encoder[self.unk_token]		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       int		):
									if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
									return self.decoder.get(_UpperCAmelCase	,self.unk_token		)







							def 							__lowercase			(							self					:       int	,_UpperCAmelCase					:       Dict		):
									_a     :      Dict        =							[]
									_a     :      List[str]        =							''
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													out_string += self.sp_model.decode(_UpperCAmelCase		) + token
													_a     :      Optional[Any]        =							[]
											else:
													current_sub_tokens.append(_UpperCAmelCase		)
									out_string += self.sp_model.decode(_UpperCAmelCase		)
									return out_string.strip()







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       List[int]	,_UpperCAmelCase					:       Optional[List[int]] = None	,_UpperCAmelCase					:       bool = False		):

									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_UpperCAmelCase	,token_ids_a=_UpperCAmelCase	,already_has_special_tokens=_UpperCAmelCase		)

									_a     :      List[str]        =							[1] * len(self.prefix_tokens		)
									_a     :      Any        =							[1] * len(self.suffix_tokens		)
									if token_ids_a is None:
											return prefix_ones + ([0] * len(_UpperCAmelCase		)) + suffix_ones
									return prefix_ones + ([0] * len(_UpperCAmelCase		)) + ([0] * len(_UpperCAmelCase		)) + suffix_ones







							def 							__lowercase			(							self					:       Optional[int]	,_UpperCAmelCase					:       List[int]	,_UpperCAmelCase					:       Optional[List[int]] = None		):
									if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







							def 							__lowercase			(							self					:       List[Any]		):
									_a     :      Dict        =							{self.convert_ids_to_tokens(_UpperCAmelCase		): i for i in range(self.vocab_size		)}
									vocab.update(self.added_tokens_encoder		)
									return vocab







							def __getstate__(							self					:       List[str]		):
									_a     :      Dict        =							self.__dict__.copy()
									_a     :      Any        =							None
									return state







							def __setstate__(							self					:       Union[str, Any]	,_UpperCAmelCase					:       Dict		):
									_a     :      List[Any]        =							d

									# for backward compatibility
									if not hasattr(self	,'sp_model_kwargs'		):
											_a     :      Union[str, Any]        =							{}

									_a     :      Any        =							load_spm(self.spm_file	,self.sp_model_kwargs		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       str	,_UpperCAmelCase					:       Optional[str] = None		):
									_a     :      List[str]        =							Path(_UpperCAmelCase		)
									if not save_dir.is_dir():
											raise OSError(F"""{save_directory} should be a directory"""		)
									_a     :      int        =							save_dir / (
									    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
									)
									_a     :      Tuple        =							save_dir / (
									    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
									)

									save_json(self.encoder	,_UpperCAmelCase		)

									if os.path.abspath(self.spm_file		) != os.path.abspath(_UpperCAmelCase		) and os.path.isfile(self.spm_file		):
											copyfile(self.spm_file	,_UpperCAmelCase		)
									elif not os.path.isfile(self.spm_file		):
											with open(_UpperCAmelCase	,'wb'		) as fi:
													_a     :      Tuple        =							self.sp_model.serialized_model_proto()
													fi.write(_UpperCAmelCase		)

									return (str(_UpperCAmelCase		), str(_UpperCAmelCase		))







							def 							__lowercase			(							self					:       List[str]	,_UpperCAmelCase					:       List[str]	,_UpperCAmelCase					:       str = "en"	,_UpperCAmelCase					:       Optional[List[str]] = None	,_UpperCAmelCase					:       str = "ro"	,**_UpperCAmelCase					:       List[Any]	,):
									_a     :      int        =							src_lang
									_a     :      List[str]        =							tgt_lang
									self.set_src_lang_special_tokens(self.src_lang		)
									return super().prepare_seqaseq_batch(_UpperCAmelCase	,_UpperCAmelCase	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Dict	,_UpperCAmelCase					:       List[str]	,_UpperCAmelCase					:       Optional[str]	,_UpperCAmelCase					:       Optional[str]	,**_UpperCAmelCase					:       List[Any]		):
									if src_lang is None or tgt_lang is None:
											raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'		)
									_a     :      List[str]        =							src_lang
									_a     :      Optional[int]        =							self(_UpperCAmelCase	,add_special_tokens=_UpperCAmelCase	,**_UpperCAmelCase		)
									_a     :      List[str]        =							self.get_lang_id(_UpperCAmelCase		)
									_a     :      List[str]        =							tgt_lang_id
									return inputs







							def 							__lowercase			(							self					:       List[str]		):
									self.set_src_lang_special_tokens(self.src_lang		)







							def 							__lowercase			(							self					:       Any		):
									self.set_tgt_lang_special_tokens(self.tgt_lang		)







							def 							__lowercase			(							self					:       Union[str, Any]	,_UpperCAmelCase					:       str		):
									_a     :      Tuple        =							self.get_lang_token(_UpperCAmelCase		)
									_a     :      Optional[Any]        =							self.lang_token_to_id[lang_token]
									_a     :      Union[str, Any]        =							[self.cur_lang_id]
									_a     :      Union[str, Any]        =							[self.eos_token_id]







							def 							__lowercase			(							self					:       List[str]	,_UpperCAmelCase					:       str		):
									_a     :      int        =							self.get_lang_token(_UpperCAmelCase		)
									_a     :      Optional[Any]        =							self.lang_token_to_id[lang_token]
									_a     :      Dict        =							[self.cur_lang_id]
									_a     :      Optional[int]        =							[self.eos_token_id]







							def 							__lowercase			(							self					:       str	,_UpperCAmelCase					:       str		):
									return self.lang_code_to_token[lang]







							def 							__lowercase			(							self					:       List[Any]	,_UpperCAmelCase					:       str		):
									_a     :      Optional[int]        =							self.get_lang_token(_UpperCAmelCase		)
									return self.lang_token_to_id[lang_token]



def      __lowerCamelCase     (      lowerCAmelCase_						,			lowerCAmelCase_  )			->      sentencepiece.SentencePieceProcessor:
		_a     :      int        =							sentencepiece.SentencePieceProcessor(**lowerCAmelCase_  )
		spm.Load(str(lowerCAmelCase_  )  )
		return spm



def      __lowerCamelCase     (      lowerCAmelCase_  )			->      Union[Dict, List]:
		with open(lowerCAmelCase_						,			'r'  ) as f:
				return json.load(lowerCAmelCase_  )



def      __lowerCamelCase     (      lowerCAmelCase_						,			lowerCAmelCase_  )			->      None:
		with open(lowerCAmelCase_						,			'w'  ) as f:
				json.dump(lowerCAmelCase_						,			lowerCAmelCase_						,			indent=2  )









'''simple docstring'''

# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowerCAmelCase									=   {
    '''configuration_cpmant''': ['''CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CpmAntConfig'''],
    '''tokenization_cpmant''': ['''CpmAntTokenizer'''],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase									=   [
							    '''CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''CpmAntForCausalLM''',
							    '''CpmAntModel''',
							    '''CpmAntPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
							from .tokenization_cpmant import CpmAntTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_cpmant import (
														    CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    CpmAntForCausalLM,
														    CpmAntModel,
														    CpmAntPreTrainedModel,
														)


else:
							import sys

							__lowerCAmelCase									=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




