




"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase__			=		logging.get_logger(__name__)

lowercase__			=		"""▁"""

lowercase__			=		{"""vocab_file""": """sentencepiece.bpe.model"""}

lowercase__			=		{
    """vocab_file""": {
        """facebook/nllb-200-distilled-600M""": (
            """https://huggingface.co/facebook/nllb-200-distilled-600M/blob/main/sentencepiece.bpe.model"""
        ),
    }
}

lowercase__			=		{
    """facebook/nllb-200-distilled-600M""": 1024,
}

# fmt: off
lowercase__			=		["""ace_Arab""", """ace_Latn""", """acm_Arab""", """acq_Arab""", """aeb_Arab""", """afr_Latn""", """ajp_Arab""", """aka_Latn""", """amh_Ethi""", """apc_Arab""", """arb_Arab""", """ars_Arab""", """ary_Arab""", """arz_Arab""", """asm_Beng""", """ast_Latn""", """awa_Deva""", """ayr_Latn""", """azb_Arab""", """azj_Latn""", """bak_Cyrl""", """bam_Latn""", """ban_Latn""", """bel_Cyrl""", """bem_Latn""", """ben_Beng""", """bho_Deva""", """bjn_Arab""", """bjn_Latn""", """bod_Tibt""", """bos_Latn""", """bug_Latn""", """bul_Cyrl""", """cat_Latn""", """ceb_Latn""", """ces_Latn""", """cjk_Latn""", """ckb_Arab""", """crh_Latn""", """cym_Latn""", """dan_Latn""", """deu_Latn""", """dik_Latn""", """dyu_Latn""", """dzo_Tibt""", """ell_Grek""", """eng_Latn""", """epo_Latn""", """est_Latn""", """eus_Latn""", """ewe_Latn""", """fao_Latn""", """pes_Arab""", """fij_Latn""", """fin_Latn""", """fon_Latn""", """fra_Latn""", """fur_Latn""", """fuv_Latn""", """gla_Latn""", """gle_Latn""", """glg_Latn""", """grn_Latn""", """guj_Gujr""", """hat_Latn""", """hau_Latn""", """heb_Hebr""", """hin_Deva""", """hne_Deva""", """hrv_Latn""", """hun_Latn""", """hye_Armn""", """ibo_Latn""", """ilo_Latn""", """ind_Latn""", """isl_Latn""", """ita_Latn""", """jav_Latn""", """jpn_Jpan""", """kab_Latn""", """kac_Latn""", """kam_Latn""", """kan_Knda""", """kas_Arab""", """kas_Deva""", """kat_Geor""", """knc_Arab""", """knc_Latn""", """kaz_Cyrl""", """kbp_Latn""", """kea_Latn""", """khm_Khmr""", """kik_Latn""", """kin_Latn""", """kir_Cyrl""", """kmb_Latn""", """kon_Latn""", """kor_Hang""", """kmr_Latn""", """lao_Laoo""", """lvs_Latn""", """lij_Latn""", """lim_Latn""", """lin_Latn""", """lit_Latn""", """lmo_Latn""", """ltg_Latn""", """ltz_Latn""", """lua_Latn""", """lug_Latn""", """luo_Latn""", """lus_Latn""", """mag_Deva""", """mai_Deva""", """mal_Mlym""", """mar_Deva""", """min_Latn""", """mkd_Cyrl""", """plt_Latn""", """mlt_Latn""", """mni_Beng""", """khk_Cyrl""", """mos_Latn""", """mri_Latn""", """zsm_Latn""", """mya_Mymr""", """nld_Latn""", """nno_Latn""", """nob_Latn""", """npi_Deva""", """nso_Latn""", """nus_Latn""", """nya_Latn""", """oci_Latn""", """gaz_Latn""", """ory_Orya""", """pag_Latn""", """pan_Guru""", """pap_Latn""", """pol_Latn""", """por_Latn""", """prs_Arab""", """pbt_Arab""", """quy_Latn""", """ron_Latn""", """run_Latn""", """rus_Cyrl""", """sag_Latn""", """san_Deva""", """sat_Beng""", """scn_Latn""", """shn_Mymr""", """sin_Sinh""", """slk_Latn""", """slv_Latn""", """smo_Latn""", """sna_Latn""", """snd_Arab""", """som_Latn""", """sot_Latn""", """spa_Latn""", """als_Latn""", """srd_Latn""", """srp_Cyrl""", """ssw_Latn""", """sun_Latn""", """swe_Latn""", """swh_Latn""", """szl_Latn""", """tam_Taml""", """tat_Cyrl""", """tel_Telu""", """tgk_Cyrl""", """tgl_Latn""", """tha_Thai""", """tir_Ethi""", """taq_Latn""", """taq_Tfng""", """tpi_Latn""", """tsn_Latn""", """tso_Latn""", """tuk_Latn""", """tum_Latn""", """tur_Latn""", """twi_Latn""", """tzm_Tfng""", """uig_Arab""", """ukr_Cyrl""", """umb_Latn""", """urd_Arab""", """uzn_Latn""", """vec_Latn""", """vie_Latn""", """war_Latn""", """wol_Latn""", """xho_Latn""", """ydd_Hebr""", """yor_Latn""", """yue_Hant""", """zho_Hans""", """zho_Hant""", """zul_Latn"""]






class lowerCAmelCase__      (	A__		):






			'''simple docstring'''



			lowerCamelCase__ =     VOCAB_FILES_NAMES
			lowerCamelCase__ =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__ =     PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__ =     ["""input_ids""", """attention_mask"""]

			lowerCamelCase__ =     []
			lowerCamelCase__ =     []
			def __init__(    self    , lowercase    , lowercase="<s>"    , lowercase="</s>"    , lowercase="</s>"    , lowercase="<s>"    , lowercase="<unk>"    , lowercase="<pad>"    , lowercase="<mask>"    , lowercase=None    , lowercase=None    , lowercase=None    , lowercase = None    , lowercase=None    , lowercase=False    , **lowercase    , ):
							# Mask token behave like a normal word, i.e. include the space before it
							_lowerCamelCase	:     List[Any]       =       AddedToken(a__    , lstrip=a__    , rstrip=a__    ) if isinstance(a__    , a__    ) else mask_token

							_lowerCamelCase	:     Tuple       =       {} if sp_model_kwargs is None else sp_model_kwargs
							_lowerCamelCase	:     Any       =       legacy_behaviour

							super().__init__(
							    bos_token=a__    , eos_token=a__    , unk_token=a__    , sep_token=a__    , cls_token=a__    , pad_token=a__    , mask_token=a__    , tokenizer_file=a__    , src_lang=a__    , tgt_lang=a__    , additional_special_tokens=a__    , sp_model_kwargs=self.sp_model_kwargs    , legacy_behaviour=a__    , **a__    , )

							_lowerCamelCase	:     Union[str, Any]       =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(str(a__    )    )
							_lowerCamelCase	:     str       =       vocab_file
							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4   |  5   |  6   |   7  |   8  |  9
							# -------- | ------- | ------- | ------ | ------- | ---- | ---- | ---- | ---- | ---- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | 'an' | '▁n' | '▁m' | '▁t' | '▁k' | '▁a'
							# spm      | '<unk>' | '<s>'   | '</s>' | 'an'    | '▁n' | '▁m' | '▁t' | '▁k' | '▁a' | '▁s'

							# Mimic fairseq token-to-id alignment for the first 4 token
							_lowerCamelCase	:     Tuple       =       {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_lowerCamelCase	:     List[Any]       =       1

							_lowerCamelCase	:     Any       =       len(self.sp_model    )
							_lowerCamelCase	:     str       =       {
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a__    )
							}
							_lowerCamelCase	:     Tuple       =       {v: k for k, v in self.lang_code_to_id.items()}
							_lowerCamelCase	:     int       =       len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
							_lowerCamelCase	:     Tuple       =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
							_lowerCamelCase	:     Union[str, Any]       =       list(self.lang_code_to_id.keys()    )

							if additional_special_tokens is not None:
											# Only add those special tokens if they are not already there.
											self._additional_special_tokens.extend(
											    [t for t in additional_special_tokens if t not in self._additional_special_tokens]    )

							_lowerCamelCase	:     List[str]       =       src_lang if src_lang is not None else 'eng_Latn'
							_lowerCamelCase	:     List[Any]       =       self.lang_code_to_id[self._src_lang]
							_lowerCamelCase	:     Optional[int]       =       tgt_lang
							self.set_src_lang_special_tokens(self._src_lang    )
			def __getstate__(    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.__dict__.copy()
							_lowerCamelCase	:     Union[str, Any]       =       None
							_lowerCamelCase	:     Tuple       =       self.sp_model.serialized_model_proto()
							return state
			def __setstate__(    self    , lowercase    ):
							_lowerCamelCase	:     str       =       d

							# for backward compatibility
							if not hasattr(self    , 'sp_model_kwargs'    ):
											_lowerCamelCase	:     Optional[Any]       =       {}

							_lowerCamelCase	:     Optional[int]       =       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )
			@property
			def    A_       (    self    ):
							return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token
			@property
			def    A_       (    self    ):
							return self._src_lang
			@src_lang.setter
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Optional[int]       =       new_src_lang
							self.set_src_lang_special_tokens(self._src_lang    )
			def    A_       (    self    , lowercase    , lowercase = None    , lowercase = False    ):

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=a__    , token_ids_a=a__    , already_has_special_tokens=a__    )

							_lowerCamelCase	:     Any       =       [1] * len(self.prefix_tokens    )
							_lowerCamelCase	:     Optional[Any]       =       [1] * len(self.suffix_tokens    )
							if token_ids_a is None:
											return prefix_ones + ([0] * len(a__    )) + suffix_ones
							return prefix_ones + ([0] * len(a__    )) + ([0] * len(a__    )) + suffix_ones
			def    A_       (    self    , lowercase    , lowercase = None    ):
							if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
			def    A_       (    self    , lowercase    , lowercase = None    ):
							_lowerCamelCase	:     List[str]       =       [self.sep_token_id]
							_lowerCamelCase	:     Tuple       =       [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep    ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]
			def    A_       (    self    , lowercase    , lowercase    , lowercase    , lowercase    , **lowercase    ):
							if src_lang is None or tgt_lang is None:
											raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'    )
							_lowerCamelCase	:     List[str]       =       src_lang
							_lowerCamelCase	:     List[str]       =       self(a__    , add_special_tokens=a__    , return_tensors=a__    , **a__    )
							_lowerCamelCase	:     Any       =       self.convert_tokens_to_ids(a__    )
							_lowerCamelCase	:     Tuple       =       tgt_lang_id
							return inputs
			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       {self.convert_ids_to_tokens(a__    ): i for i in range(self.vocab_size    )}
							vocab.update(self.added_tokens_encoder    )
							return vocab
			def    A_       (    self    , lowercase    ):
							return self.sp_model.encode(a__    , out_type=a__    )
			def    A_       (    self    , lowercase    ):
							if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
							_lowerCamelCase	:     List[Any]       =       self.sp_model.PieceToId(a__    )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
			def    A_       (    self    , lowercase    ):
							if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset    )
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Optional[int]       =       ''.join(a__    ).replace(a__    , ' '    ).strip()
							return out_string
			def    A_       (    self    , lowercase    , lowercase = None    ):
							if not os.path.isdir(a__    ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
											return
							_lowerCamelCase	:     str       =       os.path.join(
							    a__    , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(a__    ) and os.path.isfile(self.vocab_file    ):
											copyfile(self.vocab_file    , a__    )
							elif not os.path.isfile(self.vocab_file    ):
											with open(a__    , 'wb'    ) as fi:
															_lowerCamelCase	:     int       =       self.sp_model.serialized_model_proto()
															fi.write(a__    )

							return (out_vocab_file,)
			def    A_       (    self    , lowercase    , lowercase = "eng_Latn"    , lowercase = None    , lowercase = "fra_Latn"    , **lowercase    , ):
							_lowerCamelCase	:     Union[str, Any]       =       src_lang
							_lowerCamelCase	:     int       =       tgt_lang
							return super().prepare_seqaseq_batch(a__    , a__    , **a__    )
			def    A_       (    self    ):
							return self.set_src_lang_special_tokens(self.src_lang    )
			def    A_       (    self    ):
							return self.set_tgt_lang_special_tokens(self.tgt_lang    )
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     List[Any]       =       self.lang_code_to_id[src_lang]
							if self.legacy_behaviour:
											_lowerCamelCase	:     Optional[int]       =       []
											_lowerCamelCase	:     List[str]       =       [self.eos_token_id, self.cur_lang_code]
							else:
											_lowerCamelCase	:     Dict       =       [self.cur_lang_code]
											_lowerCamelCase	:     List[Any]       =       [self.eos_token_id]






			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Union[str, Any]       =       self.lang_code_to_id[lang]
							if self.legacy_behaviour:
											_lowerCamelCase	:     int       =       []
											_lowerCamelCase	:     Any       =       [self.eos_token_id, self.cur_lang_code]
							else:
											_lowerCamelCase	:     List[Any]       =       [self.cur_lang_code]
											_lowerCamelCase	:     Union[str, Any]       =       [self.eos_token_id]




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_lowercase    	=		{
    "configuration_layoutlmv2": ["LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP", "LayoutLMv2Config"],
    "processing_layoutlmv2": ["LayoutLMv2Processor"],
    "tokenization_layoutlmv2": ["LayoutLMv2Tokenizer"],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		["LayoutLMv2TokenizerFast"]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		["LayoutLMv2FeatureExtractor"]
   _lowercase    	=		["LayoutLMv2ImageProcessor"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _lowercase    	=		[
       "LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST",
       "LayoutLMv2ForQuestionAnswering",
       "LayoutLMv2ForSequenceClassification",
       "LayoutLMv2ForTokenClassification",
       "LayoutLMv2Layer",
       "LayoutLMv2Model",
       "LayoutLMv2PreTrainedModel",
   ]

if TYPE_CHECKING:
   from .configuration_layoutlmva import LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMvaConfig
   from .processing_layoutlmva import LayoutLMvaProcessor
   from .tokenization_layoutlmva import LayoutLMvaTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor, LayoutLMvaImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_layoutlmva import (
          LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST,
          LayoutLMvaForQuestionAnswering,
          LayoutLMvaForSequenceClassification,
          LayoutLMvaForTokenClassification,
          LayoutLMvaLayer,
          LayoutLMvaModel,
          LayoutLMvaPreTrainedModel,
      )
else:
   import sys

   _lowercase    	=		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
