


from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers.models.layoutlm.modeling_tf_layoutlm import (
         TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFLayoutLMForMaskedLM,
         TFLayoutLMForQuestionAnswering,
         TFLayoutLMForSequenceClassification,
         TFLayoutLMForTokenClassification,
         TFLayoutLMModel,
     )







class 		_snake_case  :




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase=13					,  _lowerCamelCase=7					,  _lowerCamelCase=True					,  _lowerCamelCase=True					,  _lowerCamelCase=True					,  _lowerCamelCase=True					,  _lowerCamelCase=99					,  _lowerCamelCase=32					,  _lowerCamelCase=2					,  _lowerCamelCase=4					,  _lowerCamelCase=37					,  _lowerCamelCase="gelu"					,  _lowerCamelCase=0.1					,  _lowerCamelCase=0.1					,  _lowerCamelCase=512					,  _lowerCamelCase=16					,  _lowerCamelCase=2					,  _lowerCamelCase=0.02					,  _lowerCamelCase=3					,  _lowerCamelCase=4					,  _lowerCamelCase=None					,  _lowerCamelCase=1000					,  ):
       a						:str 							=       parent
       a						:str 							=       batch_size
       a						:List[Any] 							=       seq_length
       a						:Union[str, Any] 							=       is_training
       a						:str 							=       use_input_mask
       a						:Tuple 							=       use_token_type_ids
       a						:Optional[int] 							=       use_labels
       a						:Union[str, Any] 							=       vocab_size
       a						:Optional[Any] 							=       hidden_size
       a						:Any 							=       num_hidden_layers
       a						:Optional[int] 							=       num_attention_heads
       a						:Tuple 							=       intermediate_size
       a						:Dict 							=       hidden_act
       a						:str 							=       hidden_dropout_prob
       a						:List[Any] 							=       attention_probs_dropout_prob
       a						:List[Any] 							=       max_position_embeddings
       a						:List[str] 							=       type_vocab_size
       a						:List[Any] 							=       type_sequence_label_size
       a						:Union[str, Any] 							=       initializer_range
       a						:Optional[Any] 							=       num_labels
       a						:Optional[int] 							=       num_choices
       a						:Union[str, Any] 							=       scope
       a						:List[str] 							=       range_bbox




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:str 							=       ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size    )

       # convert bbox to numpy since TF does not support item assignment
       a						:Union[str, Any] 							=       ids_tensor([self.batch_size, self.seq_length, 4]					,  self.range_bbox    ).numpy()
       # Ensure that bbox is legal
       for i in range(bbox.shape[0]    ):
             for j in range(bbox.shape[1]    ):
                   if bbox[i, j, 3] < bbox[i, j, 1]:
                         a						:List[Any] 							=       bbox[i, j, 3]
                         a						:List[str] 							=       bbox[i, j, 1]
                         a						:List[str] 							=       t
                   if bbox[i, j, 2] < bbox[i, j, 0]:
                         a						:Dict 							=       bbox[i, j, 2]
                         a						:Dict 							=       bbox[i, j, 0]
                         a						:Any 							=       t
       a						:Optional[Any] 							=       tf.convert_to_tensor(_lowerCamelCase    )

       a						:int 							=       None
       if self.use_input_mask:
             a						:List[Any] 							=       random_attention_mask([self.batch_size, self.seq_length]    )

       a						:Optional[int] 							=       None
       if self.use_token_type_ids:
             a						:Optional[Any] 							=       ids_tensor([self.batch_size, self.seq_length]					,  self.type_vocab_size    )

       a						:List[Any] 							=       None
       a						:List[Any] 							=       None
       a						:List[Any] 							=       None
       if self.use_labels:
             a						:Tuple 							=       ids_tensor([self.batch_size]					,  self.type_sequence_label_size    )
             a						:Dict 							=       ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels    )
             a						:List[str] 							=       ids_tensor([self.batch_size]					,  self.num_choices    )

       a						:List[Any] 							=       LayoutLMConfig(
           vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  type_vocab_size=self.type_vocab_size					,  initializer_range=self.initializer_range					,  )

       return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Optional[int] 							=       TFLayoutLMModel(config=_lowerCamelCase    )

       a						:Dict 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase    )
       a						:Union[str, Any] 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  token_type_ids=_lowerCamelCase    )
       a						:Union[str, Any] 							=       model(_lowerCamelCase					,  _lowerCamelCase    )
       self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)    )
       self.parent.assertEqual(result.pooler_output.shape					,  (self.batch_size, self.hidden_size)    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:List[str] 							=       TFLayoutLMForMaskedLM(config=_lowerCamelCase    )

       a						:int 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase					,  labels=_lowerCamelCase    )
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Optional[int] 							=       self.num_labels
       a						:List[Any] 							=       TFLayoutLMForSequenceClassification(config=_lowerCamelCase    )

       a						:int 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase    )
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_labels)    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:int 							=       self.num_labels
       a						:Optional[int] 							=       TFLayoutLMForTokenClassification(config=_lowerCamelCase    )

       a						:int 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase					,  labels=_lowerCamelCase    )
       self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.num_labels)    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase					,  _lowerCamelCase    ):
       a						:Optional[Any] 							=       TFLayoutLMForQuestionAnswering(config=_lowerCamelCase    )

       a						:Optional[int] 							=       model(_lowerCamelCase					,  _lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase    )
       self.parent.assertEqual(result.start_logits.shape					,  (self.batch_size, self.seq_length)    )
       self.parent.assertEqual(result.end_logits.shape					,  (self.batch_size, self.seq_length)    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[str] 							=       self.prepare_config_and_inputs()
       (
           (
           a
       )							,				(
           a
       )							,				(
           a
       )							,				(
           a
       )							,				(
           a
       )							,				(
           a
       )							,				(
           a
       )							,				(
           a
       )							,				
       )						:List[Any] 							=       config_and_inputs
       a						:Union[str, Any] 							=       {
           '''input_ids''': input_ids,
           '''bbox''': bbox,
           '''token_type_ids''': token_type_ids,
           '''attention_mask''': input_mask,
       }
       return config, inputs_dict







@require_tf
class 		_snake_case  (      _snake_case						,    _snake_case						,    unittest.TestCase							):
 SCREAMING_SNAKE_CASE__      =       (
     (
         TFLayoutLMModel,
         TFLayoutLMForMaskedLM,
         TFLayoutLMForTokenClassification,
         TFLayoutLMForSequenceClassification,
         TFLayoutLMForQuestionAnswering,
     )
     if is_tf_available()
     else ()
 )
 SCREAMING_SNAKE_CASE__      =       (
     {
         'feature-extraction': TFLayoutLMModel,
         'fill-mask': TFLayoutLMForMaskedLM,
         'text-classification': TFLayoutLMForSequenceClassification,
         'token-classification': TFLayoutLMForTokenClassification,
         'zero-shot': TFLayoutLMForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 SCREAMING_SNAKE_CASE__      =       False
 SCREAMING_SNAKE_CASE__      =       True
 SCREAMING_SNAKE_CASE__      =       10




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Dict 							=       TFLayoutLMModelTester(self    )
       a						:Dict 							=       ConfigTester(self					,  config_class=_lowerCamelCase					,  hidden_size=37    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       self.config_tester.run_common_tests()




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:str 							=       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[Any] 							=       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[Any] 							=       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:int 							=       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Optional[Any] 							=       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase    )




 @slow
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             a						:str 							=       TFLayoutLMModel.from_pretrained(_lowerCamelCase    )
             self.assertIsNotNone(_lowerCamelCase    )




 @unittest.skip('''Onnx compliancy broke with TF 2.10'''    )
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       pass







def 				__lowerCamelCase						( ):




      """simple docstring"""





      a						:Tuple 							=       tf.convert_to_tensor([[101,1019,1014,1016,1037,1_2849,4747,1004,1_4246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,1_1300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,1_9274,2772,6205,2_7814,1_6147,1_6147,4343,2047,1_0283,1_0969,1_4389,1012,2338,102]]						)  # noqa: E231
      a						:Any 							=       tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]						)  # noqa: E231
      a						:List[str] 							=       tf.convert_to_tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]						)  # noqa: E231
      a						:List[str] 							=       tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]						)  # noqa: E231
      # these are sequence labels (i.e. at the token level)
      a						:Any 							=       tf.convert_to_tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]						)  # noqa: E231
      # fmt: on

      return input_ids, attention_mask, bbox, token_type_ids, labels







@require_tf
class 		_snake_case  (      unittest.TestCase							):




 @slow
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:List[Any] 							=       TFLayoutLMModel.from_pretrained('''microsoft/layoutlm-base-uncased'''    )

       a							,				a							,				a							,				a							,				a						:Optional[Any] 							=       prepare_layoutlm_batch_inputs()

       # forward pass
       a						:Tuple 							=       model(input_ids=_lowerCamelCase					,  bbox=_lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase    )

       # test the sequence output on [0, :3, :3]
       a						:List[str] 							=       tf.convert_to_tensor(
           [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]					,  )

       self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]					,  _lowerCamelCase					,  atol=1e-3    )    )

       # test the pooled output on [1, :3]
       a						:List[str] 							=       tf.convert_to_tensor([-0.6580, -0.0214, 0.8552]    )

       self.assertTrue(np.allclose(outputs.pooler_output[1, :3]					,  _lowerCamelCase					,  atol=1e-3    )    )




 @slow
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       # initialize model with randomly initialized sequence classification head
       a						:str 							=       TFLayoutLMForSequenceClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''					,  num_labels=2    )

       a							,				a							,				a							,				a							,				a						:List[str] 							=       prepare_layoutlm_batch_inputs()

       # forward pass
       a						:List[Any] 							=       model(
           input_ids=_lowerCamelCase					,  bbox=_lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase					,  labels=tf.convert_to_tensor([1, 1]    )					,  )

       # test whether we get a loss as a scalar
       a						:Union[str, Any] 							=       outputs.loss
       a						:Optional[Any] 							=       (2,)
       self.assertEqual(loss.shape					,  _lowerCamelCase    )

       # test the shape of the logits
       a						:Any 							=       outputs.logits
       a						:Tuple 							=       (2, 2)
       self.assertEqual(logits.shape					,  _lowerCamelCase    )




 @slow
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       # initialize model with randomly initialized token classification head
       a						:Dict 							=       TFLayoutLMForTokenClassification.from_pretrained('''microsoft/layoutlm-base-uncased'''					,  num_labels=13    )

       a							,				a							,				a							,				a							,				a						:Dict 							=       prepare_layoutlm_batch_inputs()

       # forward pass
       a						:List[Any] 							=       model(
           input_ids=_lowerCamelCase					,  bbox=_lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase					,  labels=_lowerCamelCase    )

       # test the shape of the logits
       a						:Optional[Any] 							=       outputs.logits
       a						:List[Any] 							=       tf.convert_to_tensor((2, 25, 13)    )
       self.assertEqual(logits.shape					,  _lowerCamelCase    )




 @slow
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       # initialize model with randomly initialized token classification head
       a						:List[Any] 							=       TFLayoutLMForQuestionAnswering.from_pretrained('''microsoft/layoutlm-base-uncased'''    )

       a							,				a							,				a							,				a							,				a						:Any 							=       prepare_layoutlm_batch_inputs()

       # forward pass
       a						:str 							=       model(input_ids=_lowerCamelCase					,  bbox=_lowerCamelCase					,  attention_mask=_lowerCamelCase					,  token_type_ids=_lowerCamelCase    )

       # test the shape of the logits
       a						:Optional[int] 							=       tf.convert_to_tensor((2, 25)    )
       self.assertEqual(outputs.start_logits.shape					,  _lowerCamelCase    )
       self.assertEqual(outputs.end_logits.shape					,  _lowerCamelCase    )









import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case				:    Dict       				=     logging.get_logger(__name__)

snake_case				:    Tuple       				=     '''▁'''

snake_case				:    Any       				=     {'''vocab_file''': '''sentencepiece.bpe.model'''}

snake_case				:    Tuple       				=     {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

snake_case				:    int       				=     {
    '''xlm-roberta-base''': 5_12,
    '''xlm-roberta-large''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_12,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_12,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_12,
}







class 		_snake_case  (      _snake_case							):
 SCREAMING_SNAKE_CASE__      =       VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__      =       ['input_ids', 'attention_mask']




 def __init__(		self					,  _lowerCamelCase					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="</s>"					,  _lowerCamelCase="<s>"					,  _lowerCamelCase="<unk>"					,  _lowerCamelCase="<pad>"					,  _lowerCamelCase="<mask>"					,  _lowerCamelCase = None					,  **_lowerCamelCase					,  ):
       # Mask token behave like a normal word, i.e. include the space before it
       a						:Optional[int] 							=       AddedToken(_lowerCamelCase					,  lstrip=_lowerCamelCase					,  rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase					,  _lowerCamelCase    ) else mask_token

       a						:int 							=       {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=_lowerCamelCase					,  eos_token=_lowerCamelCase					,  unk_token=_lowerCamelCase					,  sep_token=_lowerCamelCase					,  cls_token=_lowerCamelCase					,  pad_token=_lowerCamelCase					,  mask_token=_lowerCamelCase					,  sp_model_kwargs=self.sp_model_kwargs					,  **_lowerCamelCase					,  )

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(_lowerCamelCase    )    )
       a						:str 							=       vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       a						:Tuple 							=       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       a						:List[str] 							=       1

       a						:Dict 							=       len(self.sp_model    ) + self.fairseq_offset
       a						:List[Any] 							=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




 def __getstate__(		self    ):
       a						:List[str] 							=       self.__dict__.copy()
       a						:Optional[int] 							=       None
       a						:int 							=       self.sp_model.serialized_model_proto()
       return state




 def __setstate__(		self					,  _lowerCamelCase    ):
       a						:Union[str, Any] 							=       d

       # for backward compatibility
       if not hasattr(self					,  '''sp_model_kwargs'''    ):
             a						:Union[str, Any] 							=       {}

       a						:Optional[int] 							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):

       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       a						:List[Any] 							=       [self.cls_token_id]
       a						:Dict 							=       [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None					,  _lowerCamelCase = False    ):

       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_lowerCamelCase					,  token_ids_a=_lowerCamelCase					,  already_has_special_tokens=_lowerCamelCase    )

       if token_ids_a is None:
             return [1] + ([0] * len(_lowerCamelCase    )) + [1]
       return [1] + ([0] * len(_lowerCamelCase    )) + [1, 1] + ([0] * len(_lowerCamelCase    )) + [1]




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       a						:int 							=       [self.sep_token_id]
       a						:int 							=       [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




 @property
 def        SCREAMING_SNAKE_CASE__				(		self    ):
       return len(self.sp_model    ) + self.fairseq_offset + 1  # Add the <mask> token




 def        SCREAMING_SNAKE_CASE__				(		self    ):
       a						:Any 							=       {self.convert_ids_to_tokens(_lowerCamelCase    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       return self.sp_model.encode(_lowerCamelCase					,  out_type=_lowerCamelCase    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
       a						:Optional[Any] 							=       self.sp_model.PieceToId(_lowerCamelCase    )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase    ):
       a						:Tuple 							=       ''''''.join(_lowerCamelCase    ).replace(_lowerCamelCase					,  ''' '''    ).strip()
       return out_string




 def        SCREAMING_SNAKE_CASE__				(		self					,  _lowerCamelCase					,  _lowerCamelCase = None    ):
       if not os.path.isdir(_lowerCamelCase    ):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
             return
       a						:int 							=       os.path.join(
           _lowerCamelCase					,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(_lowerCamelCase    ) and os.path.isfile(self.vocab_file    ):
             copyfile(self.vocab_file					,  _lowerCamelCase    )
       elif not os.path.isfile(self.vocab_file    ):
             with open(_lowerCamelCase					,  '''wb'''    ) as fi:
                   a						:List[Any] 							=       self.sp_model.serialized_model_proto()
                   fi.write(_lowerCamelCase    )

       return (out_vocab_file,)





