




"""simple docstring"""







import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
  import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_	     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	     =       {'vocab_file': 'vocab.txt'}

SCREAMING_SNAKE_CASE_	     =       {
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

SCREAMING_SNAKE_CASE_	     =       {
    'openbmb/cpm-ant-10b': 1024,
}





def      __snake_case (				_lowercase    ):


    """simple docstring"""







    UpperCamelCase              =	collections.OrderedDict()
    with open(_lowercase			,'''r'''			,encoding='''utf-8'''    ) as reader:
        UpperCamelCase              =	reader.readlines()
    for index, token in enumerate(_lowercase    ):
        UpperCamelCase              =	token.rstrip('''\n'''    )
        UpperCamelCase              =	index
    return vocab

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   def __init__(		self       ,					lowerCamelCase_       ,					lowerCamelCase_="<unk>"       ,					lowerCamelCase_=2_0_0)				->   Any:
       UpperCamelCase              =	vocab
       UpperCamelCase              =	unk_token
       UpperCamelCase              =	max_input_chars_per_word







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Union[str, Any]:
       UpperCamelCase              =	list(lowerCamelCase_)
       if len(lowerCamelCase_) > self.max_input_chars_per_word:
           return [self.unk_token]

       UpperCamelCase              =	0
       UpperCamelCase              =	[]
       while start < len(lowerCamelCase_):
           UpperCamelCase              =	len(lowerCamelCase_)
           UpperCamelCase              =	None
           while start < end:
               UpperCamelCase              =	''''''.join(chars[start:end])
               if substr in self.vocab:
                   UpperCamelCase              =	substr
                   break
               end -= 1
           if cur_substr is None:
               sub_tokens.append(self.unk_token)
               start += 1
           else:
               sub_tokens.append(lowerCamelCase_)
               UpperCamelCase              =	end

       return sub_tokens

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   A_				      =       VOCAB_FILES_NAMES
   A_				      =       PRETRAINED_VOCAB_FILES_MAP
   A_				      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A_				      =       ['''input_ids''', '''attention_mask''']
   A_				      =       False







   def __init__(		self       ,					lowerCamelCase_       ,					lowerCamelCase_="<d>"       ,					lowerCamelCase_="</d>"       ,					lowerCamelCase_="<s>"       ,					lowerCamelCase_="</s>"       ,					lowerCamelCase_="<pad>"       ,					lowerCamelCase_="<unk>"       ,					lowerCamelCase_="</n>"       ,					lowerCamelCase_="</_>"       ,					lowerCamelCase_="left"       ,					**lowerCamelCase_       ,					)				->   List[str]:
       requires_backends(self       ,					['''jieba'''])
       super().__init__(
           bod_token=lowerCamelCase_       ,					eod_token=lowerCamelCase_       ,					bos_token=lowerCamelCase_       ,					eos_token=lowerCamelCase_       ,					pad_token=lowerCamelCase_       ,					unk_token=lowerCamelCase_       ,					line_token=lowerCamelCase_       ,					space_token=lowerCamelCase_       ,					padding_side=lowerCamelCase_       ,					**lowerCamelCase_       ,					)
       UpperCamelCase              =	bod_token
       UpperCamelCase              =	eod_token
       UpperCamelCase              =	load_vocab(lowerCamelCase_)
       UpperCamelCase              =	self.encoder[space_token]
       UpperCamelCase              =	self.encoder[line_token]

       del self.encoder[space_token]
       del self.encoder[line_token]

       UpperCamelCase              =	collections.OrderedDict(sorted(self.encoder.items()       ,					key=lambda lowerCamelCase_: x[1]))
       UpperCamelCase              =	{v: k for k, v in self.encoder.items()}

       UpperCamelCase              =	WordpieceTokenizer(vocab=self.encoder       ,					unk_token=self.unk_token)







   @property
   def   UpperCAmelCase__			(		self)				->   Dict:
       return self.encoder[self.bod_token]







   @property
   def   UpperCAmelCase__			(		self)				->   str:
       return self.encoder[self.eod_token]







   @property
   def   UpperCAmelCase__			(		self)				->   List[Any]:
       return self.encoder["\n"]







   @property
   def   UpperCAmelCase__			(		self)				->   int:
       return len(self.encoder)







   def   UpperCAmelCase__			(		self)				->   Dict:
       return dict(self.encoder       ,					**self.added_tokens_encoder)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Any:
       UpperCamelCase              =	[]
       for x in jieba.cut(lowerCamelCase_       ,					cut_all=lowerCamelCase_):
           output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCamelCase_))
       return output_tokens







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					**lowerCamelCase_)				->   Tuple:
       UpperCamelCase              =	[i for i in token_ids if i >= 0]
       UpperCamelCase              =	[
           x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
       ]
       return super()._decode(lowerCamelCase_       ,					**lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Dict:
       return token in self.encoder







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   str:
       return "".join(lowerCamelCase_)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Optional[int]:
       return self.encoder.get(lowerCamelCase_       ,					self.encoder.get(self.unk_token))







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_)				->   Dict:
       return self.decoder.get(lowerCamelCase_       ,					self.unk_token)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   Tuple[str]:
       if os.path.isdir(lowerCamelCase_):
           UpperCamelCase              =	os.path.join(
               lowerCamelCase_       ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
       else:
           UpperCamelCase              =	(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
       UpperCamelCase              =	0
       if " " in self.encoder:
           UpperCamelCase              =	self.encoder[''' ''']
           del self.encoder[" "]
       if "\n" in self.encoder:
           UpperCamelCase              =	self.encoder['''\n''']
           del self.encoder["\n"]
       UpperCamelCase              =	collections.OrderedDict(sorted(self.encoder.items()       ,					key=lambda lowerCamelCase_: x[1]))
       with open(lowerCamelCase_       ,					'''w'''       ,					encoding='''utf-8''') as writer:
           for token, token_index in self.encoder.items():
               if index != token_index:
                   logger.warning(
                       F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                       ''' Please check that the vocabulary is not corrupted!''')
                   UpperCamelCase              =	token_index
               writer.write(token + '''\n''')
               index += 1
       return (vocab_file,)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       if token_ids_a is None:
           return [self.bos_token_id] + token_ids_a
       return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None       ,					lowerCamelCase_ = False)				->   List[int]:

       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=lowerCamelCase_       ,					token_ids_a=lowerCamelCase_       ,					already_has_special_tokens=lowerCamelCase_)

       if token_ids_a is not None:
           return [1] + ([0] * len(lowerCamelCase_)) + [1] + ([0] * len(lowerCamelCase_))
       return [1] + ([0] * len(lowerCamelCase_))
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a  :      List[Any]            =    logging.get_logger(__name__)

a  :      Union[str, Any]            =    {
    """bigcode/gpt_bigcode-santacoder""": """https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json""",
}

class 					_lowercase    (     UpperCAmelCase__			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				List[str]          =    'gpt_bigcode'
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    ['past_key_values']
    SCREAMING_SNAKE_CASE:				Tuple          =    {
        'hidden_size': 'n_embd',
        'max_position_embeddings': 'n_positions',
        'num_attention_heads': 'n_head',
        'num_hidden_layers': 'n_layer',
    }




    def __init__(    self       ,						lowerCamelCase__=50_257       ,						lowerCamelCase__=1_024       ,						lowerCamelCase__=768       ,						lowerCamelCase__=12       ,						lowerCamelCase__=12       ,						lowerCamelCase__=None       ,						lowerCamelCase__="gelu_pytorch_tanh"       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=0.1       ,						lowerCamelCase__=1E-5       ,						lowerCamelCase__=0.0_2       ,						lowerCamelCase__=True       ,						lowerCamelCase__=True       ,						lowerCamelCase__=50_256       ,						lowerCamelCase__=50_256       ,						lowerCamelCase__=True       ,						lowerCamelCase__=True       ,						lowerCamelCase__=True       ,						**lowerCamelCase__       ,						):
       lowerCAmelCase_:							List[Any]       =	vocab_size
       lowerCAmelCase_:							List[str]       =	n_positions
       lowerCAmelCase_:							Union[str, Any]       =	n_embd
       lowerCAmelCase_:							Union[str, Any]       =	n_layer
       lowerCAmelCase_:							Union[str, Any]       =	n_head
       lowerCAmelCase_:							List[Any]       =	n_inner
       lowerCAmelCase_:							List[Any]       =	activation_function
       lowerCAmelCase_:							Optional[int]       =	resid_pdrop
       lowerCAmelCase_:							Union[str, Any]       =	embd_pdrop
       lowerCAmelCase_:							Any       =	attn_pdrop
       lowerCAmelCase_:							Union[str, Any]       =	layer_norm_epsilon
       lowerCAmelCase_:							Optional[Any]       =	initializer_range
       lowerCAmelCase_:							int       =	scale_attn_weights
       lowerCAmelCase_:							int       =	use_cache
       lowerCAmelCase_:							int       =	attention_softmax_in_fpaa
       lowerCAmelCase_:							int       =	scale_attention_softmax_in_fpaa
       lowerCAmelCase_:							Union[str, Any]       =	multi_query

       lowerCAmelCase_:							Union[str, Any]       =	bos_token_id
       lowerCAmelCase_:							List[str]       =	eos_token_id

       super().__init__(bos_token_id=lowerCamelCase__       ,						eos_token_id=lowerCamelCase__       ,						**lowerCamelCase__ )