




import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__magic_name__  :  List[Any]					=       logging.get_logger(__name__)

__magic_name__  :  Dict					=       """▁"""

__magic_name__  :  str					=       {
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

__magic_name__  :  Optional[Any]					=       {
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

__magic_name__  :  Tuple					=       {
    """facebook/m2m100_418M""": 10_24,
}

# fmt: off
__magic_name__  :  int					=       {
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}





class     SCREAMING_SNAKE_CASE__		(_a			):
     lowercase_ :      Any				= VOCAB_FILES_NAMES
     lowercase_ :      int				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase_ :      Optional[Any]				= PRETRAINED_VOCAB_FILES_MAP
     lowercase_ :      List[str]				= ["input_ids", "attention_mask"]

     lowercase_ :      List[int]				= []
     lowercase_ :      List[int]				= []
     def __init__(						self	:		List[Any]       ,						__lowerCamelCase	:		List[str]       ,						__lowerCamelCase	:		List[Any]       ,						__lowerCamelCase	:		Optional[int]=None       ,						__lowerCamelCase	:		int=None       ,						__lowerCamelCase	:		List[Any]="<s>"       ,						__lowerCamelCase	:		Union[str, Any]="</s>"       ,						__lowerCamelCase	:		List[str]="</s>"       ,						__lowerCamelCase	:		Dict="<pad>"       ,						__lowerCamelCase	:		Any="<unk>"       ,						__lowerCamelCase	:		str="m2m100"       ,						__lowerCamelCase	:		Optional[Dict[str, Any]] = None       ,						__lowerCamelCase	:		List[Any]=8       ,						**__lowerCamelCase	:		Union[str, Any]       ,						):



       """simple docstring"""
       lowerCAmelCase__										=      {} if sp_model_kwargs is None else sp_model_kwargs

       lowerCAmelCase__										=      language_codes
       lowerCAmelCase__										=      FAIRSEQ_LANGUAGE_CODES[language_codes]
       lowerCAmelCase__										=      {lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

       lowerCAmelCase__										=      kwargs.get('''additional_special_tokens'''       ,						[]						)
       kwargs["additional_special_tokens"] += [
           self.get_lang_token(__lowerCamelCase						)
           for lang_code in fairseq_language_code
           if self.get_lang_token(__lowerCamelCase						) not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           src_lang=__lowerCamelCase       ,						tgt_lang=__lowerCamelCase       ,						bos_token=__lowerCamelCase       ,						eos_token=__lowerCamelCase       ,						sep_token=__lowerCamelCase       ,						unk_token=__lowerCamelCase       ,						pad_token=__lowerCamelCase       ,						language_codes=__lowerCamelCase       ,						sp_model_kwargs=self.sp_model_kwargs       ,						num_madeup_words=__lowerCamelCase       ,						**__lowerCamelCase       ,						)

       lowerCAmelCase__										=      vocab_file
       lowerCAmelCase__										=      load_json(__lowerCamelCase						)
       lowerCAmelCase__										=      {v: k for k, v in self.encoder.items()}
       lowerCAmelCase__										=      spm_file
       lowerCAmelCase__										=      load_spm(__lowerCamelCase       ,						self.sp_model_kwargs						)

       lowerCAmelCase__										=      len(self.encoder						)

       lowerCAmelCase__										=      {
           self.get_lang_token(__lowerCamelCase						): self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase						)
       }
       lowerCAmelCase__										=      {lang_code: self.encoder_size + i for i, lang_code in enumerate(__lowerCamelCase						)}
       lowerCAmelCase__										=      {v: k for k, v in self.lang_token_to_id.items()}

       lowerCAmelCase__										=      src_lang if src_lang is not None else '''en'''
       lowerCAmelCase__										=      tgt_lang
       lowerCAmelCase__										=      self.get_lang_id(self._src_lang						)
       self.set_src_lang_special_tokens(self._src_lang						)

       lowerCAmelCase__										=      num_madeup_words
     @property
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       return len(self.encoder						) + len(self.lang_token_to_id						)
     @property
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       return self._src_lang
     @src_lang.setter
     def        A__      (						self	:		Union[str, Any]       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      new_src_lang
       self.set_src_lang_special_tokens(self._src_lang						)
     def        A__      (						self	:		Any       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       return self.sp_model.encode(__lowerCamelCase       ,						out_type=__lowerCamelCase						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       if token in self.lang_token_to_id:
         return self.lang_token_to_id[token]
       return self.encoder.get(__lowerCamelCase       ,						self.encoder[self.unk_token]						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		int						):



       """simple docstring"""
       if index in self.id_to_lang_token:
         return self.id_to_lang_token[index]
       return self.decoder.get(__lowerCamelCase       ,						self.unk_token						)
     def        A__      (						self	:		Union[str, Any]       ,						__lowerCamelCase	:		Optional[int]						):



       """simple docstring"""
       lowerCAmelCase__										=      []
       lowerCAmelCase__										=      ''''''
       for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
           out_string += self.sp_model.decode(__lowerCamelCase						) + token
           lowerCAmelCase__										=      []
         else:
           current_sub_tokens.append(__lowerCamelCase						)
       out_string += self.sp_model.decode(__lowerCamelCase						)
       return out_string.strip()
     def        A__      (						self	:		Any       ,						__lowerCamelCase	:		List[int]       ,						__lowerCamelCase	:		Optional[List[int]] = None       ,						__lowerCamelCase	:		bool = False						):



       """simple docstring"""

       if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=__lowerCamelCase       ,						token_ids_a=__lowerCamelCase       ,						already_has_special_tokens=__lowerCamelCase						)

       lowerCAmelCase__										=      [1] * len(self.prefix_tokens						)
       lowerCAmelCase__										=      [1] * len(self.suffix_tokens						)
       if token_ids_a is None:
         return prefix_ones + ([0] * len(__lowerCamelCase						)) + suffix_ones
       return prefix_ones + ([0] * len(__lowerCamelCase						)) + ([0] * len(__lowerCamelCase						)) + suffix_ones
     def        A__      (						self	:		int       ,						__lowerCamelCase	:		List[int]       ,						__lowerCamelCase	:		Optional[List[int]] = None						):



       """simple docstring"""
       if token_ids_a is None:
         return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      {self.convert_ids_to_tokens(__lowerCamelCase						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab
     def __getstate__(						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      self.__dict__.copy()
       lowerCAmelCase__										=      None
       return state
     def __setstate__(						self	:		List[Any]       ,						__lowerCamelCase	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      d

       # for backward compatibility
       if not hasattr(self       ,						'''sp_model_kwargs'''						):
         lowerCAmelCase__										=      {}

       lowerCAmelCase__										=      load_spm(self.spm_file       ,						self.sp_model_kwargs						)
     def        A__      (						self	:		str       ,						__lowerCamelCase	:		str       ,						__lowerCamelCase	:		Optional[str] = None						):



       """simple docstring"""
       lowerCAmelCase__										=      Path(__lowerCamelCase						)
       if not save_dir.is_dir():
         raise OSError(F"""{save_directory} should be a directory"""						)
       lowerCAmelCase__										=      save_dir / (
           (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
       )
       lowerCAmelCase__										=      save_dir / (
           (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
       )

       save_json(self.encoder       ,						__lowerCamelCase						)

       if os.path.abspath(self.spm_file						) != os.path.abspath(__lowerCamelCase						) and os.path.isfile(self.spm_file						):
         copyfile(self.spm_file       ,						__lowerCamelCase						)
       elif not os.path.isfile(self.spm_file						):
         with open(__lowerCamelCase       ,						'''wb'''						) as fi:
           lowerCAmelCase__										=      self.sp_model.serialized_model_proto()
           fi.write(__lowerCamelCase						)

       return (str(__lowerCamelCase						), str(__lowerCamelCase						))
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		List[str]       ,						__lowerCamelCase	:		str = "en"       ,						__lowerCamelCase	:		Optional[List[str]] = None       ,						__lowerCamelCase	:		str = "ro"       ,						**__lowerCamelCase	:		Optional[Any]       ,						):



       """simple docstring"""
       lowerCAmelCase__										=      src_lang
       lowerCAmelCase__										=      tgt_lang
       self.set_src_lang_special_tokens(self.src_lang						)
       return super().prepare_seqaseq_batch(__lowerCamelCase       ,						__lowerCamelCase       ,						**__lowerCamelCase						)
     def        A__      (						self	:		List[Any]       ,						__lowerCamelCase	:		str       ,						__lowerCamelCase	:		Optional[str]       ,						__lowerCamelCase	:		Optional[str]       ,						**__lowerCamelCase	:		Union[str, Any]						):



       """simple docstring"""
       if src_lang is None or tgt_lang is None:
         raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''						)
       lowerCAmelCase__										=      src_lang
       lowerCAmelCase__										=      self(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase       ,						**__lowerCamelCase						)
       lowerCAmelCase__										=      self.get_lang_id(__lowerCamelCase						)
       lowerCAmelCase__										=      tgt_lang_id
       return inputs
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       self.set_src_lang_special_tokens(self.src_lang						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       self.set_tgt_lang_special_tokens(self.tgt_lang						)
     def        A__      (						self	:		Optional[Any]       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_lang_token(__lowerCamelCase						)
       lowerCAmelCase__										=      self.lang_token_to_id[lang_token]
       lowerCAmelCase__										=      [self.cur_lang_id]
       lowerCAmelCase__										=      [self.eos_token_id]
     def        A__      (						self	:		List[str]       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_lang_token(__lowerCamelCase						)
       lowerCAmelCase__										=      self.lang_token_to_id[lang_token]
       lowerCAmelCase__										=      [self.cur_lang_id]
       lowerCAmelCase__										=      [self.eos_token_id]
     def        A__      (						self	:		Any       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       return self.lang_code_to_token[lang]



     def        A__      (						self	:		Optional[int]       ,						__lowerCamelCase	:		str						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_lang_token(__lowerCamelCase						)
       return self.lang_token_to_id[lang_token]



def 		a_							( __lowerCAmelCase   , __lowerCAmelCase				):
  lowerCAmelCase__										=      sentencepiece.SentencePieceProcessor(**__lowerCAmelCase				)
  spm.Load(str(__lowerCAmelCase				)				)
  return spm
def 		a_							( __lowerCAmelCase				):
  with open(__lowerCAmelCase   , '''r'''				) as f:
    return json.load(__lowerCAmelCase				)




def 		a_							( __lowerCAmelCase   , __lowerCAmelCase				):
  with open(__lowerCAmelCase   , '''w'''				) as f:
    json.dump(__lowerCAmelCase   , __lowerCAmelCase   , indent=2				)







from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__magic_name__  :  Optional[Any]					=       {
    """configuration_informer""": [
        """INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """InformerConfig""",
    ],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __magic_name__  :  int					=       [
          """INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """InformerForPrediction""",
          """InformerModel""",
          """InformerPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_informer import INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, InformerConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_informer import (
                INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                InformerForPrediction,
                InformerModel,
                InformerPreTrainedModel,
            )

else:
      import sys

      __magic_name__  :  Any					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

