



import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class    UpperCAmelCase__:

   '''simple docstring'''




   def __init__(    self	:      List[Any]      ,					lowerCAmelCase	:      Dict      ,					lowerCAmelCase	:      Optional[Any]=13      ,					lowerCAmelCase	:      Optional[int]=7      ,					lowerCAmelCase	:      Any=True      ,					lowerCAmelCase	:      List[Any]=True      ,					lowerCAmelCase	:      Any=False      ,					lowerCAmelCase	:      Optional[int]=True      ,					lowerCAmelCase	:      Optional[int]=99      ,					lowerCAmelCase	:      int=32      ,					lowerCAmelCase	:      List[Any]=5      ,					lowerCAmelCase	:      int=4      ,					lowerCAmelCase	:      str=37      ,					lowerCAmelCase	:      List[Any]="gelu"      ,					lowerCAmelCase	:      Union[str, Any]=0.1      ,					lowerCAmelCase	:      Union[str, Any]=0.1      ,					lowerCAmelCase	:      str=5_12      ,					lowerCAmelCase	:      int=16      ,					lowerCAmelCase	:      int=2      ,					lowerCAmelCase	:      Optional[Any]=0.02      ,					lowerCAmelCase	:      int=3      ,					lowerCAmelCase	:      List[Any]=4      ,					lowerCAmelCase	:      Dict=None      ,					)						->					Optional[Any]:




         """simple docstring"""

         lowercase__  =	parent
         lowercase__  =	batch_size
         lowercase__  =	seq_length
         lowercase__  =	is_training
         lowercase__  =	use_input_mask
         lowercase__  =	use_token_type_ids
         lowercase__  =	use_labels
         lowercase__  =	vocab_size
         lowercase__  =	hidden_size
         lowercase__  =	num_hidden_layers
         lowercase__  =	num_attention_heads
         lowercase__  =	intermediate_size
         lowercase__  =	hidden_act
         lowercase__  =	hidden_dropout_prob
         lowercase__  =	attention_probs_dropout_prob
         lowercase__  =	max_position_embeddings
         lowercase__  =	type_vocab_size
         lowercase__  =	type_sequence_label_size
         lowercase__  =	initializer_range
         lowercase__  =	num_labels
         lowercase__  =	num_choices
         lowercase__  =	scope


   def       UpperCAmelCase		(    self	:      List[str])						->					Dict:




         """simple docstring"""

         lowercase__  =	ids_tensor([self.batch_size, self.seq_length]      ,					self.vocab_size)

         lowercase__  =	None
         if self.use_input_mask:
               lowercase__  =	random_attention_mask([self.batch_size, self.seq_length])

         lowercase__  =	None
         if self.use_token_type_ids:
               lowercase__  =	ids_tensor([self.batch_size, self.seq_length]      ,					self.type_vocab_size)

         lowercase__  =	None
         lowercase__  =	None
         lowercase__  =	None
         if self.use_labels:
               lowercase__  =	ids_tensor([self.batch_size]      ,					self.type_sequence_label_size)
               lowercase__  =	ids_tensor([self.batch_size, self.seq_length]      ,					self.num_labels)
               lowercase__  =	ids_tensor([self.batch_size]      ,					self.num_choices)

         lowercase__  =	self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


   def       UpperCAmelCase		(    self	:      Dict)						->					Union[str, Any]:




         """simple docstring"""

         return OpenLlamaConfig(
             vocab_size=self.vocab_size      ,					hidden_size=self.hidden_size      ,					num_hidden_layers=self.num_hidden_layers      ,					num_attention_heads=self.num_attention_heads      ,					intermediate_size=self.intermediate_size      ,					hidden_act=self.hidden_act      ,					hidden_dropout_prob=self.hidden_dropout_prob      ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,					max_position_embeddings=self.max_position_embeddings      ,					type_vocab_size=self.type_vocab_size      ,					is_decoder=lowerCAmelCase      ,					initializer_range=self.initializer_range      ,					use_stable_embedding=lowerCAmelCase      ,					)


   def       UpperCAmelCase		(    self	:      str      ,					lowerCAmelCase	:      Optional[int]      ,					lowerCAmelCase	:      Optional[Any]      ,					lowerCAmelCase	:      Tuple      ,					lowerCAmelCase	:      List[str]      ,					lowerCAmelCase	:      str      ,					lowerCAmelCase	:      List[Any]      ,					lowerCAmelCase	:      List[str])						->					Tuple:




         """simple docstring"""

         lowercase__  =	OpenLlamaModel(config=lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase)
         lowercase__  =	model(lowerCAmelCase)
         self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size))


   def       UpperCAmelCase		(    self	:      Optional[Any]      ,					lowerCAmelCase	:      Union[str, Any]      ,					lowerCAmelCase	:      List[str]      ,					lowerCAmelCase	:      int      ,					lowerCAmelCase	:      Dict      ,					lowerCAmelCase	:      Dict      ,					lowerCAmelCase	:      List[str]      ,					lowerCAmelCase	:      Any      ,					lowerCAmelCase	:      List[str]      ,					lowerCAmelCase	:      Optional[int]      ,					)						->					Union[str, Any]:




         """simple docstring"""

         lowercase__  =	True
         lowercase__  =	OpenLlamaModel(lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(
             lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					encoder_hidden_states=lowerCAmelCase      ,					encoder_attention_mask=lowerCAmelCase      ,					)
         lowercase__  =	model(
             lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					encoder_hidden_states=lowerCAmelCase      ,					)
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase)
         self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size))


   def       UpperCAmelCase		(    self	:      Optional[Any]      ,					lowerCAmelCase	:      Tuple      ,					lowerCAmelCase	:      List[Any]      ,					lowerCAmelCase	:      List[str]      ,					lowerCAmelCase	:      Any      ,					lowerCAmelCase	:      List[Any]      ,					lowerCAmelCase	:      int      ,					lowerCAmelCase	:      Tuple      ,					lowerCAmelCase	:      int      ,					lowerCAmelCase	:      Tuple      ,					)						->					List[Any]:




         """simple docstring"""

         lowercase__  =	OpenLlamaForCausalLM(config=lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					labels=lowerCAmelCase)
         self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, self.seq_length, self.vocab_size))


   def       UpperCAmelCase		(    self	:      Tuple      ,					lowerCAmelCase	:      Dict      ,					lowerCAmelCase	:      int      ,					lowerCAmelCase	:      Optional[Any]      ,					lowerCAmelCase	:      Union[str, Any]      ,					lowerCAmelCase	:      str      ,					lowerCAmelCase	:      int      ,					lowerCAmelCase	:      Any      ,					lowerCAmelCase	:      List[Any]      ,					lowerCAmelCase	:      Tuple      ,					)						->					str:




         """simple docstring"""

         lowercase__  =	True
         lowercase__  =	True
         lowercase__  =	OpenLlamaForCausalLM(config=lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()

         # first forward pass
         lowercase__  =	model(
             lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					encoder_hidden_states=lowerCAmelCase      ,					encoder_attention_mask=lowerCAmelCase      ,					use_cache=lowerCAmelCase      ,					)
         lowercase__  =	outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         lowercase__  =	ids_tensor((self.batch_size, 3)      ,					config.vocab_size)
         lowercase__  =	ids_tensor((self.batch_size, 3)      ,					vocab_size=2)

         # append to next input_ids and
         lowercase__  =	torch.cat([input_ids, next_tokens]      ,					dim=-1)
         lowercase__  =	torch.cat([input_mask, next_mask]      ,					dim=-1)

         lowercase__  =	model(
             lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					encoder_hidden_states=lowerCAmelCase      ,					encoder_attention_mask=lowerCAmelCase      ,					output_hidden_states=lowerCAmelCase      ,					)['hidden_states'][0]
         lowercase__  =	model(
             lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					encoder_hidden_states=lowerCAmelCase      ,					encoder_attention_mask=lowerCAmelCase      ,					past_key_values=lowerCAmelCase      ,					output_hidden_states=lowerCAmelCase      ,					)['hidden_states'][0]

         # select random slice
         lowercase__  =	ids_tensor((1,)      ,					output_from_past.shape[-1]).item()
         lowercase__  =	output_from_no_past[:, -3:, random_slice_idx].detach()
         lowercase__  =	output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(lowerCAmelCase      ,					lowerCAmelCase      ,					atol=1E-3))



   def       UpperCAmelCase		(    self	:      Any)						->					Any:




         """simple docstring"""

         lowercase__  =	self.prepare_config_and_inputs()
         (
             (
             lowercase__
         ),						(
             lowercase__
         ),						(
             lowercase__
         ),						(
             lowercase__
         ),						(
             lowercase__
         ),						(
             lowercase__
         ),						(
             lowercase__
         ),						
         )  =	config_and_inputs
         lowercase__  =	{'input_ids': input_ids, 'attention_mask': input_mask}
         return config, inputs_dict



@require_torch
class    UpperCAmelCase__(				lowerCamelCase  ,				lowerCamelCase  ,				lowerCamelCase  ,				unittest.TestCase						):

   '''simple docstring'''




   A     :	int												=     (
       (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
   )
   A     :	Dict												=     (OpenLlamaForCausalLM,) if is_torch_available() else ()
   A     :	str												=     (
       {
           "feature-extraction": OpenLlamaModel,
           "text-classification": OpenLlamaForSequenceClassification,
           "text-generation": OpenLlamaForCausalLM,
           "zero-shot": OpenLlamaForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )
   A     :	Optional[Any]												=     False
   A     :	Tuple												=     False


   def       UpperCAmelCase		(    self	:      List[Any])						->					Tuple:




         """simple docstring"""

         lowercase__  =	OpenLlamaModelTester(self)
         lowercase__  =	ConfigTester(self      ,					config_class=lowerCAmelCase      ,					hidden_size=37)


   def       UpperCAmelCase		(    self	:      Optional[int])						->					Tuple:




         """simple docstring"""

         self.config_tester.run_common_tests()


   def       UpperCAmelCase		(    self	:      List[str])						->					Optional[Any]:




         """simple docstring"""

         lowercase__  =	self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      Optional[Any])						->					Optional[Any]:




         """simple docstring"""

         lowercase__  =	self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
               lowercase__  =	type
               self.model_tester.create_and_check_model(*lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      Optional[Any])						->					List[Any]:




         """simple docstring"""

         lowercase__,						lowercase__  =	self.model_tester.prepare_config_and_inputs_for_common()
         lowercase__  =	3
         lowercase__  =	input_dict['input_ids']
         lowercase__  =	input_ids.ne(1).to(lowerCAmelCase)
         lowercase__  =	ids_tensor([self.model_tester.batch_size]      ,					self.model_tester.type_sequence_label_size)
         lowercase__  =	OpenLlamaForSequenceClassification(lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					labels=lowerCAmelCase)
         self.assertEqual(result.logits.shape      ,					(self.model_tester.batch_size, self.model_tester.num_labels))


   def       UpperCAmelCase		(    self	:      Tuple)						->					Union[str, Any]:




         """simple docstring"""

         lowercase__,						lowercase__  =	self.model_tester.prepare_config_and_inputs_for_common()
         lowercase__  =	3
         lowercase__  =	'single_label_classification'
         lowercase__  =	input_dict['input_ids']
         lowercase__  =	input_ids.ne(1).to(lowerCAmelCase)
         lowercase__  =	ids_tensor([self.model_tester.batch_size]      ,					self.model_tester.type_sequence_label_size)
         lowercase__  =	OpenLlamaForSequenceClassification(lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					labels=lowerCAmelCase)
         self.assertEqual(result.logits.shape      ,					(self.model_tester.batch_size, self.model_tester.num_labels))


   def       UpperCAmelCase		(    self	:      Dict)						->					Any:




         """simple docstring"""

         lowercase__,						lowercase__  =	self.model_tester.prepare_config_and_inputs_for_common()
         lowercase__  =	3
         lowercase__  =	'multi_label_classification'
         lowercase__  =	input_dict['input_ids']
         lowercase__  =	input_ids.ne(1).to(lowerCAmelCase)
         lowercase__  =	ids_tensor(
             [self.model_tester.batch_size, config.num_labels]      ,					self.model_tester.type_sequence_label_size).to(torch.float)
         lowercase__  =	OpenLlamaForSequenceClassification(lowerCAmelCase)
         model.to(lowerCAmelCase)
         model.eval()
         lowercase__  =	model(lowerCAmelCase      ,					attention_mask=lowerCAmelCase      ,					labels=lowerCAmelCase)
         self.assertEqual(result.logits.shape      ,					(self.model_tester.batch_size, self.model_tester.num_labels))


   @unittest.skip('Open-Llama buffers include complex numbers, which breaks this test')
   def       UpperCAmelCase		(    self	:      Optional[int])						->					Any:




         """simple docstring"""

         pass



   @parameterized.expand([('linear',), ('dynamic',)])
   def       UpperCAmelCase		(    self	:      Tuple      ,					lowerCAmelCase	:      Optional[int])						->					int:




         """simple docstring"""

         lowercase__,						lowercase__  =	self.model_tester.prepare_config_and_inputs_for_common()
         lowercase__  =	ids_tensor([1, 10]      ,					config.vocab_size)
         lowercase__  =	ids_tensor([1, int(config.max_position_embeddings * 1.5)]      ,					config.vocab_size)

         set_seed(42)  # Fixed seed at init time so the two models get the same random weights
         lowercase__  =	OpenLlamaModel(lowerCAmelCase)
         original_model.to(lowerCAmelCase)
         original_model.eval()
         lowercase__  =	original_model(lowerCAmelCase).last_hidden_state
         lowercase__  =	original_model(lowerCAmelCase).last_hidden_state

         set_seed(42)  # Fixed seed at init time so the two models get the same random weights
         lowercase__  =	{'type': scaling_type, 'factor': 10.0}
         lowercase__  =	OpenLlamaModel(lowerCAmelCase)
         scaled_model.to(lowerCAmelCase)
         scaled_model.eval()
         lowercase__  =	scaled_model(lowerCAmelCase).last_hidden_state
         lowercase__  =	scaled_model(lowerCAmelCase).last_hidden_state

         # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
         # maximum sequence length, so the outputs for the short input should match.
         if scaling_type == "dynamic":
               self.assertTrue(torch.allclose(lowerCAmelCase      ,					lowerCAmelCase      ,					atol=1E-5))
         else:
               self.assertFalse(torch.allclose(lowerCAmelCase      ,					lowerCAmelCase      ,					atol=1E-5))

         # The output should be different for long inputs
         self.assertFalse(torch.allclose(lowerCAmelCase      ,					lowerCAmelCase      ,					atol=1E-5))





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__   :					Optional[int] 						=      logging.get_logger(__name__)
a__   :					Dict 						=      {"vocab_file": "spiece.model"}

a__   :					str 						=      {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

a__   :					Union[str, Any] 						=      {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

a__   :					Optional[int] 						=      "▁"



class    UpperCAmelCase__(				lowerCamelCase						):

   '''simple docstring'''




   A     :	Tuple												=     VOCAB_FILES_NAMES
   A     :	Dict												=     PRETRAINED_VOCAB_FILES_MAP
   A     :	str												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


   def __init__(    self	:      List[str]      ,					lowerCAmelCase	:      Tuple      ,					lowerCAmelCase	:      Optional[int]=True      ,					lowerCAmelCase	:      List[Any]=True      ,					lowerCAmelCase	:      str=False      ,					lowerCAmelCase	:      str="[CLS]"      ,					lowerCAmelCase	:      Optional[int]="[SEP]"      ,					lowerCAmelCase	:      int="<unk>"      ,					lowerCAmelCase	:      str="[SEP]"      ,					lowerCAmelCase	:      Dict="<pad>"      ,					lowerCAmelCase	:      int="[CLS]"      ,					lowerCAmelCase	:      Tuple="[MASK]"      ,					lowerCAmelCase	:      Optional[Dict[str, Any]] = None      ,					**lowerCAmelCase	:      List[str]      ,					)						->					None:




         """simple docstring"""

         lowercase__  =	(
             AddedToken(lowerCAmelCase      ,					lstrip=lowerCAmelCase      ,					rstrip=lowerCAmelCase      ,					normalized=lowerCAmelCase)
             if isinstance(lowerCAmelCase      ,					lowerCAmelCase)
             else mask_token
         )

         lowercase__  =	{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=lowerCAmelCase      ,					remove_space=lowerCAmelCase      ,					keep_accents=lowerCAmelCase      ,					bos_token=lowerCAmelCase      ,					eos_token=lowerCAmelCase      ,					unk_token=lowerCAmelCase      ,					sep_token=lowerCAmelCase      ,					pad_token=lowerCAmelCase      ,					cls_token=lowerCAmelCase      ,					mask_token=lowerCAmelCase      ,					sp_model_kwargs=self.sp_model_kwargs      ,					**lowerCAmelCase      ,					)

         lowercase__  =	do_lower_case
         lowercase__  =	remove_space
         lowercase__  =	keep_accents
         lowercase__  =	vocab_file

         lowercase__  =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(lowerCAmelCase)


   @property
   def       UpperCAmelCase		(    self	:      Any)						->					Optional[int]:




         """simple docstring"""

         return len(self.sp_model)


   def       UpperCAmelCase		(    self	:      Optional[int])						->					Any:




         """simple docstring"""

         lowercase__  =	{self.convert_ids_to_tokens(lowerCAmelCase): i for i in range(self.vocab_size)}
         vocab.update(self.added_tokens_encoder)
         return vocab


   def __getstate__(    self	:      int)						->					List[str]:




         """simple docstring"""

         lowercase__  =	self.__dict__.copy()
         lowercase__  =	None
         return state


   def __setstate__(    self	:      int      ,					lowerCAmelCase	:      str)						->					Any:




         """simple docstring"""

         lowercase__  =	d

         # for backward compatibility
         if not hasattr(self      ,					'sp_model_kwargs'):
               lowercase__  =	{}

         lowercase__  =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(self.vocab_file)


   def       UpperCAmelCase		(    self	:      Dict      ,					lowerCAmelCase	:      int)						->					List[str]:




         """simple docstring"""

         if self.remove_space:
               lowercase__  =	' '.join(inputs.strip().split())
         else:
               lowercase__  =	inputs
         lowercase__  =	outputs.replace('``'      ,					'"').replace('\'\''      ,					'"')

         if not self.keep_accents:
               lowercase__  =	unicodedata.normalize('NFKD'      ,					lowerCAmelCase)
               lowercase__  =	''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase)])
         if self.do_lower_case:
               lowercase__  =	outputs.lower()

         return outputs


   def       UpperCAmelCase		(    self	:      Any      ,					lowerCAmelCase	:      str)						->					List[str]:




         """simple docstring"""

         lowercase__  =	self.preprocess_text(lowerCAmelCase)
         lowercase__  =	self.sp_model.encode(lowerCAmelCase      ,					out_type=lowerCAmelCase)
         lowercase__  =	[]
         for piece in pieces:
               if len(lowerCAmelCase) > 1 and piece[-1] == str(',') and piece[-2].isdigit():
                     lowercase__  =	self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase      ,					''))
                     if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                           if len(cur_pieces[0]) == 1:
                                 lowercase__  =	cur_pieces[1:]
                           else:
                                 lowercase__  =	cur_pieces[0][1:]
                     cur_pieces.append(piece[-1])
                     new_pieces.extend(lowerCAmelCase)
               else:
                     new_pieces.append(lowerCAmelCase)

         return new_pieces


   def       UpperCAmelCase		(    self	:      Any      ,					lowerCAmelCase	:      int)						->					int:




         """simple docstring"""

         return self.sp_model.PieceToId(lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      List[str]      ,					lowerCAmelCase	:      Any)						->					Tuple:




         """simple docstring"""

         return self.sp_model.IdToPiece(lowerCAmelCase)


   def       UpperCAmelCase		(    self	:      Tuple      ,					lowerCAmelCase	:      Optional[Any])						->					List[Any]:




         """simple docstring"""

         lowercase__  =	[]
         lowercase__  =	''
         lowercase__  =	False
         for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                     if not prev_is_special:
                           out_string += " "
                     out_string += self.sp_model.decode(lowerCAmelCase) + token
                     lowercase__  =	True
                     lowercase__  =	[]
               else:
                     current_sub_tokens.append(lowerCAmelCase)
                     lowercase__  =	False
         out_string += self.sp_model.decode(lowerCAmelCase)
         return out_string.strip()


   def       UpperCAmelCase		(    self	:      Union[str, Any]      ,					lowerCAmelCase	:      List[int]      ,					lowerCAmelCase	:      Optional[List[int]] = None)						->					List[int]:




         """simple docstring"""

         lowercase__  =	[self.sep_token_id]
         lowercase__  =	[self.cls_token_id]
         if token_ids_a is None:
               return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep


   def       UpperCAmelCase		(    self	:      int      ,					lowerCAmelCase	:      List[int]      ,					lowerCAmelCase	:      Optional[List[int]] = None      ,					lowerCAmelCase	:      bool = False)						->					List[int]:




         """simple docstring"""

         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase      ,					token_ids_a=lowerCAmelCase      ,					already_has_special_tokens=lowerCAmelCase)

         if token_ids_a is not None:
               return [1] + ([0] * len(lowerCAmelCase)) + [1] + ([0] * len(lowerCAmelCase)) + [1]
         return [1] + ([0] * len(lowerCAmelCase)) + [1]


   def       UpperCAmelCase		(    self	:      Union[str, Any]      ,					lowerCAmelCase	:      List[int]      ,					lowerCAmelCase	:      Optional[List[int]] = None)						->					List[int]:




         """simple docstring"""

         lowercase__  =	[self.sep_token_id]
         lowercase__  =	[self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
         return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



   def       UpperCAmelCase		(    self	:      Dict      ,					lowerCAmelCase	:      str      ,					lowerCAmelCase	:      Optional[str] = None)						->					Tuple[str]:




         """simple docstring"""

         if not os.path.isdir(lowerCAmelCase):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
               return
         lowercase__  =	os.path.join(
             lowerCAmelCase      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

         if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file      ,					lowerCAmelCase)
         elif not os.path.isfile(self.vocab_file):
               with open(lowerCAmelCase      ,					'wb') as fi:
                     lowercase__  =	self.sp_model.serialized_model_proto()
                     fi.write(lowerCAmelCase)

         return (out_vocab_file,)
