
import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_ 				=			logging.get_logger(__name__)

lowercase_ 				=			{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowercase_ 				=			{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowercase_ 				=			{
    """ctrl""": 256,
}

lowercase_ 				=			{
    """Pregnancy""": 168_629,
    """Christianity""": 7_675,
    """Explain""": 106_423,
    """Fitness""": 63_440,
    """Saving""": 63_163,
    """Ask""": 27_171,
    """Ass""": 95_985,
    """Joke""": 163_509,
    """Questions""": 45_622,
    """Thoughts""": 49_605,
    """Retail""": 52_342,
    """Feminism""": 164_338,
    """Writing""": 11_992,
    """Atheism""": 192_263,
    """Netflix""": 48_616,
    """Computing""": 39_639,
    """Opinion""": 43_213,
    """Alone""": 44_967,
    """Funny""": 58_917,
    """Gaming""": 40_358,
    """Human""": 4_088,
    """India""": 1_331,
    """Joker""": 77_138,
    """Diet""": 36_206,
    """Legal""": 11_859,
    """Norman""": 4_939,
    """Tip""": 72_689,
    """Weight""": 52_343,
    """Movies""": 46_273,
    """Running""": 23_425,
    """Science""": 2_090,
    """Horror""": 37_793,
    """Confession""": 60_572,
    """Finance""": 12_250,
    """Politics""": 16_360,
    """Scary""": 191_985,
    """Support""": 12_654,
    """Technologies""": 32_516,
    """Teenage""": 66_160,
    """Event""": 32_769,
    """Learned""": 67_460,
    """Notion""": 182_770,
    """Wikipedia""": 37_583,
    """Books""": 6_665,
    """Extract""": 76_050,
    """Confessions""": 102_701,
    """Conspiracy""": 75_932,
    """Links""": 63_674,
    """Narcissus""": 150_425,
    """Relationship""": 54_766,
    """Relationships""": 134_796,
    """Reviews""": 41_671,
    """News""": 4_256,
    """Translation""": 26_820,
    """multilingual""": 128_406,
}


def 							a__					(							snake_case							):





  """simple docstring"""
  __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		set()
  __SCREAMING_SNAKE_CASE      :	Any					=		word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)							)
    __SCREAMING_SNAKE_CASE      :	Dict					=		char

  __SCREAMING_SNAKE_CASE      :	int					=		set(snake_case							)
  return pairs





class     __UpperCamelCase (							lowerCAmelCase__       ):






     """simple docstring"""







     lowerCAmelCase_							   =			VOCAB_FILES_NAMES
     lowerCAmelCase_							   =			PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_							   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_							   =			CONTROL_CODES



     def __init__(				self							:							Dict     ,						_A							:							Union[str, Any]     ,						_A							:							List[Any]     ,						_A							:							Union[str, Any]="<unk>"     ,						**_A							:							Any ):







       """simple docstring"""




       super().__init__(unk_token=_A     ,						**_A )

       with open(_A     ,						encoding='''utf-8''' ) as vocab_handle:
         __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		json.load(_A )
       __SCREAMING_SNAKE_CASE      :	str					=		{v: k for k, v in self.encoder.items()}
       with open(_A     ,						encoding='''utf-8''' ) as merges_handle:
         __SCREAMING_SNAKE_CASE      :	Tuple					=		merges_handle.read().split('''\n''' )[1:-1]
       __SCREAMING_SNAKE_CASE      :	List[Any]					=		[tuple(merge.split() ) for merge in merges]
       __SCREAMING_SNAKE_CASE      :	str					=		dict(zip(_A     ,						range(len(_A ) ) ) )
       __SCREAMING_SNAKE_CASE      :	int					=		{}



     @property
     def  UpperCAmelCase__							(				self							:							List[Any] ):







       """simple docstring"""




       return len(self.encoder )



     def  UpperCAmelCase__							(				self							:							Optional[Any] ):







       """simple docstring"""




       return dict(self.encoder     ,						**self.added_tokens_encoder )



     def  UpperCAmelCase__							(				self							:							Tuple     ,						_A							:							List[str] ):







       """simple docstring"""




       if token in self.cache:
         return self.cache[token]
       __SCREAMING_SNAKE_CASE      :	List[str]					=		tuple(_A )
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		tuple(list(word[:-1] ) + [word[-1] + '''</w>'''] )
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		get_pairs(_A )

       if not pairs:
         return token

       while True:
         __SCREAMING_SNAKE_CASE      :	Optional[int]					=		min(_A     ,						key=lambda _A : self.bpe_ranks.get(_A     ,						float('''inf''' ) ) )
         if bigram not in self.bpe_ranks:
           break
         __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE      :	Optional[int]					=		bigram
         __SCREAMING_SNAKE_CASE      :	Any					=		[]
         __SCREAMING_SNAKE_CASE      :	str					=		0
         while i < len(_A ):
           try:
             __SCREAMING_SNAKE_CASE      :	Optional[int]					=		word.index(_A     ,						_A )
           except ValueError:
             new_word.extend(word[i:] )
             break
           else:
             new_word.extend(word[i:j] )
             __SCREAMING_SNAKE_CASE      :	Tuple					=		j

           if word[i] == first and i < len(_A ) - 1 and word[i + 1] == second:
             new_word.append(first + second )
             i += 2
           else:
             new_word.append(word[i] )
             i += 1
         __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		tuple(_A )
         __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		new_word
         if len(_A ) == 1:
           break
         else:
           __SCREAMING_SNAKE_CASE      :	Dict					=		get_pairs(_A )
       __SCREAMING_SNAKE_CASE      :	List[str]					=		'''@@ '''.join(_A )
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		word[:-4]
       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		word
       return word



     def  UpperCAmelCase__							(				self							:							Tuple     ,						_A							:							Union[str, Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		[]

       __SCREAMING_SNAKE_CASE      :	List[Any]					=		re.findall(r'''\S+\n?'''     ,						_A )

       for token in words:
         split_tokens.extend(list(self.bpe(_A ).split(''' ''' ) ) )
       return split_tokens



     def  UpperCAmelCase__							(				self							:							int     ,						_A							:							Optional[Any] ):







       """simple docstring"""




       return self.encoder.get(_A     ,						self.encoder.get(self.unk_token ) )



     def  UpperCAmelCase__							(				self							:							Optional[int]     ,						_A							:							Union[str, Any] ):







       """simple docstring"""




       return self.decoder.get(_A     ,						self.unk_token )



     def  UpperCAmelCase__							(				self							:							str     ,						_A							:							int ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	List[str]					=		''' '''.join(_A ).replace('''@@ '''     ,						'''''' ).strip()
       return out_string




     def  UpperCAmelCase__							(				self							:							Tuple     ,						_A							:							str     ,						_A							:							Optional[str] = None ):







       """simple docstring"""




       if not os.path.isdir(_A ):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
         return
       __SCREAMING_SNAKE_CASE      :	Tuple					=		os.path.join(
           _A     ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		os.path.join(
           _A     ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

       with open(_A     ,						'''w'''     ,						encoding='''utf-8''' ) as f:
         f.write(json.dumps(self.encoder     ,						indent=2     ,						sort_keys=_A     ,						ensure_ascii=_A ) + '''\n''' )

       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		0
       with open(_A     ,						'''w'''     ,						encoding='''utf-8''' ) as writer:
         writer.write('''#version: 0.2\n''' )
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,						key=lambda _A : kv[1] ):
           if index != token_index:
             logger.warning(
                 F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                 ''' Please check that the tokenizer is not corrupted!''' )
             __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		token_index
           writer.write(''' '''.join(_A ) + '''\n''' )
           index += 1

       return vocab_file, merge_file

     # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
     #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
     #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
     #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
     #     return ''.join(tokens_generated_so_far)







from typing import TYPE_CHECKING

from ...utils import _LazyModule


__lowerCamelCase			:	str					    =	{"""tokenization_byt5""": ["""ByT5Tokenizer"""]}


if TYPE_CHECKING:
    from .tokenization_byta import ByTaTokenizer
else:
    import sys

    __lowerCamelCase			:	List[str]					    =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
