



'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A					:       Optional[int]									=     logging.get_logger(__name__)
A					:       Any									=     {'vocab_file': 'spiece.model'}

A					:       Optional[int]									=     {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A					:       int									=     {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

A					:       Dict									=     '▁'





class 				lowerCAmelCase_						(			UpperCAmelCase_		):
    __UpperCAmelCase				=			VOCAB_FILES_NAMES
    __UpperCAmelCase				=			PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





    def __init__(							self							:							Tuple,							_snake_case							:							int,							_snake_case							:							int=True,							_snake_case							:							Tuple=True,							_snake_case							:							Tuple=False,							_snake_case							:							Union[str, Any]="[CLS]",							_snake_case							:							Any="[SEP]",							_snake_case							:							Optional[int]="<unk>",							_snake_case							:							Optional[Any]="[SEP]",							_snake_case							:							Tuple="<pad>",							_snake_case							:							Optional[int]="[CLS]",							_snake_case							:							Tuple="[MASK]",							_snake_case							:							Dict = None,							**_snake_case							:							Union[str, Any],							):



       '''simple docstring'''







       snake_case       :				Union[str, Any]		    =(
           AddedToken(__a,							lstrip=__a,							rstrip=__a,							normalized=__a      )
           if isinstance(__a,							__a      )
           else mask_token
       )

       snake_case       :				Tuple		    ={} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=__a,							remove_space=__a,							keep_accents=__a,							bos_token=__a,							eos_token=__a,							unk_token=__a,							sep_token=__a,							pad_token=__a,							cls_token=__a,							mask_token=__a,							sp_model_kwargs=self.sp_model_kwargs,							**__a,							)

       snake_case       :				Optional[Any]		    =do_lower_case
       snake_case       :				Optional[Any]		    =remove_space
       snake_case       :				Any		    =keep_accents
       snake_case       :				Optional[Any]		    =vocab_file

       snake_case       :				List[Any]		    =spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(__a      )





    @property
    def 	__snake_case  (							self							:							Optional[int]      ):



       '''simple docstring'''







       return len(self.sp_model      )





    def 	__snake_case  (							self							:							str      ):



       '''simple docstring'''







       snake_case       :				int		    ={self.convert_ids_to_tokens(__a      ): i for i in range(self.vocab_size      )}
       vocab.update(self.added_tokens_encoder      )
       return vocab





    def __getstate__(							self							:							Optional[int]      ):



       '''simple docstring'''







       snake_case       :				Dict		    =self.__dict__.copy()
       snake_case       :				str		    =None
       return state





    def __setstate__(							self							:							Optional[Any],							_snake_case							:							Union[str, Any]      ):



       '''simple docstring'''







       snake_case       :				Optional[Any]		    =d

       # for backward compatibility
       if not hasattr(self,							'''sp_model_kwargs'''      ):
          snake_case       :				Union[str, Any]		    ={}

       snake_case       :				Dict		    =spm.SentencePieceProcessor(**self.sp_model_kwargs      )
       self.sp_model.Load(self.vocab_file      )





    def 	__snake_case  (							self							:							Optional[int],							_snake_case							:							List[str]      ):



       '''simple docstring'''







       if self.remove_space:
          snake_case       :				Optional[Any]		    =''' '''.join(inputs.strip().split()      )
       else:
          snake_case       :				int		    =inputs
       snake_case       :				List[Any]		    =outputs.replace('''``''',							'''\"'''      ).replace('''\'\'''',							'''\"'''      )

       if not self.keep_accents:
          snake_case       :				int		    =unicodedata.normalize('''NFKD''',							__a      )
          snake_case       :				List[str]		    =''''''.join([c for c in outputs if not unicodedata.combining(__a      )]      )
       if self.do_lower_case:
          snake_case       :				Tuple		    =outputs.lower()

       return outputs





    def 	__snake_case  (							self							:							List[Any],							_snake_case							:							List[Any]      ):



       '''simple docstring'''







       snake_case       :				int		    =self.preprocess_text(__a      )
       snake_case       :				Optional[Any]		    =self.sp_model.encode(__a,							out_type=__a      )
       snake_case       :				int		    =[]
       for piece in pieces:
          if len(__a      ) > 1 and piece[-1] == str(''','''      ) and piece[-2].isdigit():
             snake_case       :				Dict		    =self.sp_model.EncodeAsPieces(piece[:-1].replace(__a,							''''''      )      )
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                if len(cur_pieces[0]      ) == 1:
                   snake_case       :				Optional[int]		    =cur_pieces[1:]
                else:
                   snake_case       :				Union[str, Any]		    =cur_pieces[0][1:]
             cur_pieces.append(piece[-1]      )
             new_pieces.extend(__a      )
          else:
             new_pieces.append(__a      )

       return new_pieces





    def 	__snake_case  (							self							:							Tuple,							_snake_case							:							Any      ):



       '''simple docstring'''







       return self.sp_model.PieceToId(__a      )





    def 	__snake_case  (							self							:							Tuple,							_snake_case							:							Union[str, Any]      ):



       '''simple docstring'''







       return self.sp_model.IdToPiece(__a      )





    def 	__snake_case  (							self							:							List[str],							_snake_case							:							str      ):



       '''simple docstring'''







       snake_case       :				Tuple		    =[]
       snake_case       :				List[Any]		    =''''''
       snake_case       :				Dict		    =False
       for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
             if not prev_is_special:
                out_string += " "
             out_string += self.sp_model.decode(__a      ) + token
             snake_case       :				Tuple		    =True
             snake_case       :				Union[str, Any]		    =[]
          else:
             current_sub_tokens.append(__a      )
             snake_case       :				Union[str, Any]		    =False
       out_string += self.sp_model.decode(__a      )
       return out_string.strip()





    def 	__snake_case  (							self							:							Union[str, Any],							_snake_case							:							str,							_snake_case							:							List[Any] = None      ):



       '''simple docstring'''







       snake_case       :				Optional[int]		    =[self.sep_token_id]
       snake_case       :				Union[str, Any]		    =[self.cls_token_id]
       if token_ids_a is None:
          return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep





    def 	__snake_case  (							self							:							Dict,							_snake_case							:							Tuple,							_snake_case							:							Union[str, Any] = None,							_snake_case							:							List[str] = False      ):



       '''simple docstring'''







       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__a,							token_ids_a=__a,							already_has_special_tokens=__a      )

       if token_ids_a is not None:
          return [1] + ([0] * len(__a      )) + [1] + ([0] * len(__a      )) + [1]
       return [1] + ([0] * len(__a      )) + [1]





    def 	__snake_case  (							self							:							List[str],							_snake_case							:							List[str],							_snake_case							:							str = None      ):



       '''simple docstring'''







       snake_case       :				Any		    =[self.sep_token_id]
       snake_case       :				Dict		    =[self.cls_token_id]

       if token_ids_a is None:
          return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






    def 	__snake_case  (							self							:							Tuple,							_snake_case							:							Optional[Any],							_snake_case							:							Optional[Any] = None      ):



       '''simple docstring'''







       if not os.path.isdir(__a      ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
          return
       snake_case       :				int		    =os.path.join(
           __a,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

       if os.path.abspath(self.vocab_file      ) != os.path.abspath(__a      ) and os.path.isfile(self.vocab_file      ):
          copyfile(self.vocab_file,							__a      )
       elif not os.path.isfile(self.vocab_file      ):
          with open(__a,							'''wb'''      ) as fi:
             snake_case       :				List[Any]		    =self.sp_model.serialized_model_proto()
             fi.write(__a      )

       return (out_vocab_file,)










import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
         OpenAIGPTConfig,
         OpenAIGPTDoubleHeadsModel,
         OpenAIGPTForSequenceClassification,
         OpenAIGPTLMHeadModel,
         OpenAIGPTModel,
     )





class  lowerCamelCase__	:
     """simple docstring"""



     def __init__(self					,      __a					,      __a=13					,      __a=7					,      __a=True					,      __a=True					,      __a=True					,      __a=99					,      __a=32					,      __a=5					,      __a=4					,      __a=37					,      __a="gelu"					,      __a=0.1					,      __a=0.1					,      __a=5_12					,      __a=16					,      __a=2					,      __a=0.02					,      __a=3					,      __a=4					,      __a=None					,      ):






           '''simple docstring'''



           lowerCamelCase             =		parent
           lowerCamelCase             =		batch_size
           lowerCamelCase             =		seq_length
           lowerCamelCase             =		is_training
           lowerCamelCase             =		use_token_type_ids
           lowerCamelCase             =		use_labels
           lowerCamelCase             =		vocab_size
           lowerCamelCase             =		hidden_size
           lowerCamelCase             =		num_hidden_layers
           lowerCamelCase             =		num_attention_heads
           lowerCamelCase             =		intermediate_size
           lowerCamelCase             =		hidden_act
           lowerCamelCase             =		hidden_dropout_prob
           lowerCamelCase             =		attention_probs_dropout_prob
           lowerCamelCase             =		max_position_embeddings
           lowerCamelCase             =		type_vocab_size
           lowerCamelCase             =		type_sequence_label_size
           lowerCamelCase             =		initializer_range
           lowerCamelCase             =		num_labels
           lowerCamelCase             =		num_choices
           lowerCamelCase             =		scope
           lowerCamelCase             =		self.vocab_size - 1







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		ids_tensor([self.batch_size, self.seq_length]					,      self.vocab_size     )

           lowerCamelCase             =		None
           if self.use_token_type_ids:
                 lowerCamelCase             =		ids_tensor([self.batch_size, self.seq_length]					,      self.type_vocab_size     )

           lowerCamelCase             =		None
           lowerCamelCase             =		None
           lowerCamelCase             =		None
           if self.use_labels:
                 lowerCamelCase             =		ids_tensor([self.batch_size]					,      self.type_sequence_label_size     )
                 lowerCamelCase             =		ids_tensor([self.batch_size, self.seq_length]					,      self.num_labels     )
                 lowerCamelCase             =		ids_tensor([self.batch_size]					,      self.num_choices     )

           lowerCamelCase             =		OpenAIGPTConfig(
               vocab_size=self.vocab_size					,      n_embd=self.hidden_size					,      n_layer=self.num_hidden_layers					,      n_head=self.num_attention_heads					,      n_positions=self.max_position_embeddings					,      pad_token_id=self.pad_token_id					,      )

           lowerCamelCase             =		ids_tensor([self.num_hidden_layers, self.num_attention_heads]					,      2     )

           return (
               config,
               input_ids,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      *__a     ):






           '''simple docstring'''



           lowerCamelCase             =		OpenAIGPTModel(config=__a     )
           model.to(__a     )
           model.eval()

           lowerCamelCase             =		model(__a					,      token_type_ids=__a					,      head_mask=__a     )
           lowerCamelCase             =		model(__a					,      token_type_ids=__a     )
           lowerCamelCase             =		model(__a     )

           self.parent.assertEqual(result.last_hidden_state.shape					,      (self.batch_size, self.seq_length, self.hidden_size)     )







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      *__a     ):






           '''simple docstring'''



           lowerCamelCase             =		OpenAIGPTLMHeadModel(__a     )
           model.to(__a     )
           model.eval()

           lowerCamelCase             =		model(__a					,      token_type_ids=__a					,      labels=__a     )
           self.parent.assertEqual(result.loss.shape					,      ()     )
           self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      *__a     ):






           '''simple docstring'''



           lowerCamelCase             =		OpenAIGPTDoubleHeadsModel(__a     )
           model.to(__a     )
           model.eval()

           lowerCamelCase             =		model(__a					,      token_type_ids=__a					,      labels=__a     )
           self.parent.assertEqual(result.loss.shape					,      ()     )
           self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)     )







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      *__a     ):






           '''simple docstring'''



           lowerCamelCase             =		self.num_labels
           lowerCamelCase             =		OpenAIGPTForSequenceClassification(__a     )
           model.to(__a     )
           model.eval()

           lowerCamelCase             =		ids_tensor([self.batch_size]					,      self.type_sequence_label_size     )
           lowerCamelCase             =		model(__a					,      token_type_ids=__a					,      labels=__a     )
           self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.num_labels)     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.prepare_config_and_inputs()
           (
               (
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				(
               lowerCamelCase
           )      ,				
           )             =		config_and_inputs
           lowerCamelCase             =		{
               "input_ids": input_ids,
               "token_type_ids": token_type_ids,
               "head_mask": head_mask,
           }

           return config, inputs_dict





@require_torch
class  lowerCamelCase__	(    UpperCAmelCase_							,     UpperCAmelCase_							,     UpperCAmelCase_							,     unittest.TestCase):
     """simple docstring"""



     _A      =      (
         (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
         if is_torch_available()
         else ()
     )
     _A      =      (
         (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
     )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
     _A      =      (
         {
             'feature-extraction': OpenAIGPTModel,
             'text-classification': OpenAIGPTForSequenceClassification,
             'text-generation': OpenAIGPTLMHeadModel,
             'zero-shot': OpenAIGPTForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      __a     ):






           '''simple docstring'''



           if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
                 # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
                 # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
                 # tiny config could not be created.
                 return True

           return False







     def     _a   (self					,      __a					,      __a					,      __a=False     ):






           '''simple docstring'''



           lowerCamelCase             =		super()._prepare_for_class(__a					,      __a					,      return_labels=__a     )

           if return_labels:
                 if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                       lowerCamelCase             =		torch.zeros(
                           (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)					,      dtype=torch.long					,      device=__a					,      )
                       lowerCamelCase             =		inputs_dict["labels"]
                       lowerCamelCase             =		inputs_dict["labels"]
                       lowerCamelCase             =		torch.zeros(
                           (self.model_tester.batch_size, self.model_tester.num_choices)					,      dtype=torch.long					,      device=__a					,      )
                       lowerCamelCase             =		torch.zeros(
                           self.model_tester.batch_size					,      dtype=torch.long					,      device=__a     )
           return inputs_dict







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		OpenAIGPTModelTester(self     )
           lowerCamelCase             =		ConfigTester(self					,      config_class=__a					,      n_embd=37     )







     def     _a   (self     ):






           '''simple docstring'''



           self.config_tester.run_common_tests()







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_model(*__a     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_lm_head_model(*__a     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_double_lm_head_model(*__a     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*__a     )







     @slow
     def     _a   (self     ):






           '''simple docstring'''



           for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 lowerCamelCase             =		OpenAIGPTModel.from_pretrained(__a     )
                 self.assertIsNotNone(__a     )





@require_torch
class  lowerCamelCase__	(    unittest.TestCase):
     """simple docstring"""



     @slow
     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		OpenAIGPTLMHeadModel.from_pretrained("openai-gpt"     )
           model.to(__a     )
           lowerCamelCase             =		torch.tensor([[4_81, 47_35, 5_44]]					,      dtype=torch.long					,      device=__a     )  # the president is
           lowerCamelCase             =		[
               4_81,
               47_35,
               5_44,
               2_46,
               9_63,
               8_70,
               7_62,
               2_39,
               2_44,
               4_04_77,
               2_44,
               2_49,
               7_19,
               8_81,
               4_87,
               5_44,
               2_40,
               2_44,
               6_03,
               4_81,
           ]  # the president is a very good man. " \n " i\'m sure he is, " said the

           lowerCamelCase             =		model.generate(__a					,      do_sample=__a     )
           self.assertListEqual(output_ids[0].tolist()					,      __a     )