






"""simple docstring"""



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__       :				str 					=					logging.get_logger(__name__)

A__       :				str 					=					{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

A__       :				Dict 					=					{
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

A__       :				Union[str, Any] 					=					{
    "ctrl": 256,
}

A__       :				List[Any] 					=					{
    "Pregnancy": 168_629,
    "Christianity": 7_675,
    "Explain": 106_423,
    "Fitness": 63_440,
    "Saving": 63_163,
    "Ask": 27_171,
    "Ass": 95_985,
    "Joke": 163_509,
    "Questions": 45_622,
    "Thoughts": 49_605,
    "Retail": 52_342,
    "Feminism": 164_338,
    "Writing": 11_992,
    "Atheism": 192_263,
    "Netflix": 48_616,
    "Computing": 39_639,
    "Opinion": 43_213,
    "Alone": 44_967,
    "Funny": 58_917,
    "Gaming": 40_358,
    "Human": 4_088,
    "India": 1_331,
    "Joker": 77_138,
    "Diet": 36_206,
    "Legal": 11_859,
    "Norman": 4_939,
    "Tip": 72_689,
    "Weight": 52_343,
    "Movies": 46_273,
    "Running": 23_425,
    "Science": 2_090,
    "Horror": 37_793,
    "Confession": 60_572,
    "Finance": 12_250,
    "Politics": 16_360,
    "Scary": 191_985,
    "Support": 12_654,
    "Technologies": 32_516,
    "Teenage": 66_160,
    "Event": 32_769,
    "Learned": 67_460,
    "Notion": 182_770,
    "Wikipedia": 37_583,
    "Books": 6_665,
    "Extract": 76_050,
    "Confessions": 102_701,
    "Conspiracy": 75_932,
    "Links": 63_674,
    "Narcissus": 150_425,
    "Relationship": 54_766,
    "Relationships": 134_796,
    "Reviews": 41_671,
    "News": 4_256,
    "Translation": 26_820,
    "multilingual": 128_406,
}

def  _snake_case				(     lowerCamelCase__	:    Dict  )				->	str:
							lowerCamelCase_       :							List[str]									=set()
							lowerCamelCase_       :							Tuple									=word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)  )
														lowerCamelCase_       :							List[Any]									=char

							lowerCamelCase_       :							Union[str, Any]									=set(__SCREAMING_SNAKE_CASE  )
							return pairs







class    lowercase__			(							lowerCamelCase_   ):
			_UpperCAmelCase		:Optional[Any]   			= VOCAB_FILES_NAMES
			_UpperCAmelCase		:Any   			= PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:Dict   			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:Any   			= CONTROL_CODES


			def __init__(      self     :  Tuple   ,       snake_case__     :  int   ,       snake_case__     :  List[Any]   ,       snake_case__     :  int="<unk>"   ,       **snake_case__     :  Optional[int]					):
										super().__init__(unk_token=__SCREAMING_SNAKE_CASE   ,       **__SCREAMING_SNAKE_CASE					)

										with open(__SCREAMING_SNAKE_CASE   ,       encoding="utf-8"					) as vocab_handle:
																	lowerCamelCase_       :							int									=json.load(__SCREAMING_SNAKE_CASE					)
										lowerCamelCase_       :							List[Any]									={v: k for k, v in self.encoder.items()}
										with open(__SCREAMING_SNAKE_CASE   ,       encoding="utf-8"					) as merges_handle:
																	lowerCamelCase_       :							Optional[Any]									=merges_handle.read().split("\n"					)[1:-1]
										lowerCamelCase_       :							Tuple									=[tuple(merge.split()					) for merge in merges]
										lowerCamelCase_       :							Union[str, Any]									=dict(zip(__SCREAMING_SNAKE_CASE   ,       range(len(__SCREAMING_SNAKE_CASE					)					)					)					)
										lowerCamelCase_       :							str									={}


			@property
			def 	UpperCAmelCase__						(      self     :  List[Any]					):
										return len(self.encoder					)


			def 	UpperCAmelCase__						(      self     :  Optional[int]					):
										return dict(self.encoder   ,       **self.added_tokens_encoder					)


			def 	UpperCAmelCase__						(      self     :  Optional[int]   ,       snake_case__     :  int					):





										if token in self.cache:
																	return self.cache[token]
										lowerCamelCase_       :							Tuple									=tuple(__SCREAMING_SNAKE_CASE					)
										lowerCamelCase_       :							List[Any]									=tuple(list(word[:-1]					) + [word[-1] + "</w>"]					)
										lowerCamelCase_       :							Any									=get_pairs(__SCREAMING_SNAKE_CASE					)

										if not pairs:
																	return token

										while True:
																	lowerCamelCase_       :							Optional[Any]									=min(__SCREAMING_SNAKE_CASE   ,       key=lambda snake_case__					: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE   ,       float("inf"					)					)					)
																	if bigram not in self.bpe_ranks:
																								break
																	lowerCamelCase_       :							str									=bigram
																	lowerCamelCase_       :							List[str]									=[]
																	lowerCamelCase_       :							Tuple									=0
																	while i < len(__SCREAMING_SNAKE_CASE					):
																								try:
																															lowerCamelCase_       :							int									=word.index(__SCREAMING_SNAKE_CASE   ,       __SCREAMING_SNAKE_CASE					)
																								except ValueError:
																															new_word.extend(word[i:]					)
																															break
																								else:
																															new_word.extend(word[i:j]					)
																															lowerCamelCase_       :							int									=j

																								if word[i] == first and i < len(__SCREAMING_SNAKE_CASE					) - 1 and word[i + 1] == second:
																															new_word.append(first + second					)
																															i += 2
																								else:
																															new_word.append(word[i]					)
																															i += 1
																	lowerCamelCase_       :							List[Any]									=tuple(__SCREAMING_SNAKE_CASE					)
																	lowerCamelCase_       :							List[str]									=new_word
																	if len(__SCREAMING_SNAKE_CASE					) == 1:
																								break
																	else:
																								lowerCamelCase_       :							Tuple									=get_pairs(__SCREAMING_SNAKE_CASE					)
										lowerCamelCase_       :							Dict									='''@@ '''.join(__SCREAMING_SNAKE_CASE					)
										lowerCamelCase_       :							Optional[Any]									=word[:-4]
										lowerCamelCase_       :							Union[str, Any]									=word
										return word


			def 	UpperCAmelCase__						(      self     :  Dict   ,       snake_case__     :  str					):
										lowerCamelCase_       :							Union[str, Any]									=[]

										lowerCamelCase_       :							str									=re.findall(r"\S+\n?"   ,       __SCREAMING_SNAKE_CASE					)

										for token in words:
																	split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE					).split(" "					)					)					)
										return split_tokens


			def 	UpperCAmelCase__						(      self     :  int   ,       snake_case__     :  Any					):
										return self.encoder.get(__SCREAMING_SNAKE_CASE   ,       self.encoder.get(self.unk_token					)					)


			def 	UpperCAmelCase__						(      self     :  Tuple   ,       snake_case__     :  Tuple					):
										return self.decoder.get(__SCREAMING_SNAKE_CASE   ,       self.unk_token					)


			def 	UpperCAmelCase__						(      self     :  Union[str, Any]   ,       snake_case__     :  int					):
										lowerCamelCase_       :							Tuple									=''' '''.join(__SCREAMING_SNAKE_CASE					).replace("@@ "   ,       ""					).strip()
										return out_string


			def 	UpperCAmelCase__						(      self     :  Union[str, Any]   ,       snake_case__     :  Union[str, Any]   ,       snake_case__     :  List[Any] = None					):





										if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
																	return
										lowerCamelCase_       :							Tuple									=os.path.join(
										    __SCREAMING_SNAKE_CASE   ,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
										lowerCamelCase_       :							int									=os.path.join(
										    __SCREAMING_SNAKE_CASE   ,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

										with open(__SCREAMING_SNAKE_CASE   ,       "w"   ,       encoding="utf-8"					) as f:
																	f.write(json.dumps(self.encoder   ,       indent=2   ,       sort_keys=__SCREAMING_SNAKE_CASE   ,       ensure_ascii=__SCREAMING_SNAKE_CASE					) + "\n"					)

										lowerCamelCase_       :							Dict									=0
										with open(__SCREAMING_SNAKE_CASE   ,       "w"   ,       encoding="utf-8"					) as writer:
																	writer.write("#version: 0.2\n"					)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,       key=lambda snake_case__					: kv[1]					):
																								if index != token_index:
																															logger.warning(
																															    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																															    " Please check that the tokenizer is not corrupted!"					)
																															lowerCamelCase_       :							str									=token_index
																								writer.write(" ".join(__SCREAMING_SNAKE_CASE					) + "\n"					)
																								index += 1

										return vocab_file, merge_file

			# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
			#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
			#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
			#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
			#     return ''.join(tokens_generated_so_far)





'''simple docstring'''







def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int							):



						"""simple docstring"""





						lowercase_	:     Optional[int]													=       int(__SCREAMING_SNAKE_CASE							)
						if decimal in (0, 1):  # Exit cases for the recursion
												return str(__SCREAMING_SNAKE_CASE							)
						lowercase_						, lowercase_	:     List[str]													=       divmod(__SCREAMING_SNAKE_CASE      ,      2							)
						return binary_recursive(__SCREAMING_SNAKE_CASE							) + str(__SCREAMING_SNAKE_CASE							)
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: str							):



						"""simple docstring"""





						lowercase_	:     str													=       str(__SCREAMING_SNAKE_CASE							).strip()
						if not number:
												raise ValueError('''No input value was provided'''							)
						lowercase_	:     Optional[int]													=       '''-''' if number.startswith('''-'''							) else ''''''
						lowercase_	:     Union[str, Any]													=       number.lstrip('''-'''							)
						if not number.isnumeric():
												raise ValueError('''Input value is not an integer'''							)
						return F'''{negative}0b{binary_recursive(int(__SCREAMING_SNAKE_CASE							)							)}'''


if __name__ == "__main__":
							from doctest import testmod

							testmod()



