
'''simple docstring'''







import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase :  str    							=       logging.get_logger(__name__)

_lowercase :  str    							=       {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_lowercase :  Dict    							=       {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

_lowercase :  Union[str, Any]    							=       {
    "ctrl": 2_5_6,
}

_lowercase :  List[Any]    							=       {
    "Pregnancy": 1_6_8_6_2_9,
    "Christianity": 7_6_7_5,
    "Explain": 1_0_6_4_2_3,
    "Fitness": 6_3_4_4_0,
    "Saving": 6_3_1_6_3,
    "Ask": 2_7_1_7_1,
    "Ass": 9_5_9_8_5,
    "Joke": 1_6_3_5_0_9,
    "Questions": 4_5_6_2_2,
    "Thoughts": 4_9_6_0_5,
    "Retail": 5_2_3_4_2,
    "Feminism": 1_6_4_3_3_8,
    "Writing": 1_1_9_9_2,
    "Atheism": 1_9_2_2_6_3,
    "Netflix": 4_8_6_1_6,
    "Computing": 3_9_6_3_9,
    "Opinion": 4_3_2_1_3,
    "Alone": 4_4_9_6_7,
    "Funny": 5_8_9_1_7,
    "Gaming": 4_0_3_5_8,
    "Human": 4_0_8_8,
    "India": 1_3_3_1,
    "Joker": 7_7_1_3_8,
    "Diet": 3_6_2_0_6,
    "Legal": 1_1_8_5_9,
    "Norman": 4_9_3_9,
    "Tip": 7_2_6_8_9,
    "Weight": 5_2_3_4_3,
    "Movies": 4_6_2_7_3,
    "Running": 2_3_4_2_5,
    "Science": 2_0_9_0,
    "Horror": 3_7_7_9_3,
    "Confession": 6_0_5_7_2,
    "Finance": 1_2_2_5_0,
    "Politics": 1_6_3_6_0,
    "Scary": 1_9_1_9_8_5,
    "Support": 1_2_6_5_4,
    "Technologies": 3_2_5_1_6,
    "Teenage": 6_6_1_6_0,
    "Event": 3_2_7_6_9,
    "Learned": 6_7_4_6_0,
    "Notion": 1_8_2_7_7_0,
    "Wikipedia": 3_7_5_8_3,
    "Books": 6_6_6_5,
    "Extract": 7_6_0_5_0,
    "Confessions": 1_0_2_7_0_1,
    "Conspiracy": 7_5_9_3_2,
    "Links": 6_3_6_7_4,
    "Narcissus": 1_5_0_4_2_5,
    "Relationship": 5_4_7_6_6,
    "Relationships": 1_3_4_7_9_6,
    "Reviews": 4_1_6_7_1,
    "News": 4_2_5_6,
    "Translation": 2_6_8_2_0,
    "multilingual": 1_2_8_4_0_6,
}
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: Dict							):



						"""simple docstring"""





						lowercase_	:     List[str]													=       set()
						lowercase_	:     Tuple													=       word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)							)
												lowercase_	:     List[Any]													=       char

						lowercase_	:     Union[str, Any]													=       set(__SCREAMING_SNAKE_CASE							)
						return pairs






class  lowerCAmelCase__   (  lowerCamelCase_     ):
					lowerCAmelCase_        =     VOCAB_FILES_NAMES
					lowerCAmelCase_        =     PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_        =     CONTROL_CODES






					def __init__(   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE="<unk>"				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											super().__init__(unk_token=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)

											with open(__SCREAMING_SNAKE_CASE				,							encoding='''utf-8'''		) as vocab_handle:
																	lowercase_	:     int													=       json.load(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[Any]													=       {v: k for k, v in self.encoder.items()}
											with open(__SCREAMING_SNAKE_CASE				,							encoding='''utf-8'''		) as merges_handle:
																	lowercase_	:     Optional[Any]													=       merges_handle.read().split('''\n'''		)[1:-1]
											lowercase_	:     Tuple													=       [tuple(merge.split()		) for merge in merges]
											lowercase_	:     Union[str, Any]													=       dict(zip(__SCREAMING_SNAKE_CASE				,							range(len(__SCREAMING_SNAKE_CASE		)		)		)		)
											lowercase_	:     str													=       {}






					@property
					def 	_snake_case    (   self		):


											"""simple docstring"""





											return len(self.encoder		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											return dict(self.encoder				,							**self.added_tokens_encoder		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.cache:
																	return self.cache[token]
											lowercase_	:     Tuple													=       tuple(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[Any]													=       tuple(list(word[:-1]		) + [word[-1] + '''</w>''']		)
											lowercase_	:     Any													=       get_pairs(__SCREAMING_SNAKE_CASE		)

											if not pairs:
																	return token

											while True:
																	lowercase_	:     Optional[Any]													=       min(__SCREAMING_SNAKE_CASE				,							key=lambda __SCREAMING_SNAKE_CASE		: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE				,							float('''inf'''		)		)		)
																	if bigram not in self.bpe_ranks:
																							break
																	lowercase_						, lowercase_	:     str													=       bigram
																	lowercase_	:     List[str]													=       []
																	lowercase_	:     Tuple													=       0
																	while i < len(__SCREAMING_SNAKE_CASE		):
																							try:
																													lowercase_	:     int													=       word.index(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		)
																							except ValueError:
																													new_word.extend(word[i:]		)
																													break
																							else:
																													new_word.extend(word[i:j]		)
																													lowercase_	:     int													=       j

																							if word[i] == first and i < len(__SCREAMING_SNAKE_CASE		) - 1 and word[i + 1] == second:
																													new_word.append(first + second		)
																													i += 2
																							else:
																													new_word.append(word[i]		)
																													i += 1
																	lowercase_	:     List[Any]													=       tuple(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     List[str]													=       new_word
																	if len(__SCREAMING_SNAKE_CASE		) == 1:
																							break
																	else:
																							lowercase_	:     Tuple													=       get_pairs(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Dict													=       '''@@ '''.join(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Optional[Any]													=       word[:-4]
											lowercase_	:     Union[str, Any]													=       word
											return word






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Union[str, Any]													=       []

											lowercase_	:     str													=       re.findall(R'''\S+\n?'''				,							__SCREAMING_SNAKE_CASE		)

											for token in words:
																	split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE		).split(''' '''		)		)		)
											return split_tokens






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.encoder.get(__SCREAMING_SNAKE_CASE				,							self.encoder.get(self.unk_token		)		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.decoder.get(__SCREAMING_SNAKE_CASE				,							self.unk_token		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       ''' '''.join(__SCREAMING_SNAKE_CASE		).replace('''@@ '''				,							''''''		).strip()
											return out_string






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_	:     Tuple													=       os.path.join(
											    __SCREAMING_SNAKE_CASE				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
											lowercase_	:     int													=       os.path.join(
											    __SCREAMING_SNAKE_CASE				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

											with open(__SCREAMING_SNAKE_CASE				,							'''w'''				,							encoding='''utf-8'''		) as f:
																	f.write(json.dumps(self.encoder				,							indent=2				,							sort_keys=__SCREAMING_SNAKE_CASE				,							ensure_ascii=__SCREAMING_SNAKE_CASE		) + '''\n'''		)

											lowercase_	:     Dict													=       0
											with open(__SCREAMING_SNAKE_CASE				,							'''w'''				,							encoding='''utf-8'''		) as writer:
																	writer.write('''#version: 0.2\n'''		)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,							key=lambda __SCREAMING_SNAKE_CASE		: kv[1]		):
																							if index != token_index:
																													logger.warning(
																													    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																													    ''' Please check that the tokenizer is not corrupted!'''		)
																													lowercase_	:     str													=       token_index
																							writer.write(''' '''.join(__SCREAMING_SNAKE_CASE		) + '''\n'''		)
																							index += 1

											return vocab_file, merge_file

					# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
					#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
					#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
					#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
					#     return ''.join(tokens_generated_so_far)





'''simple docstring'''







_lowercase :  int    							=       [sum(int(c, 1_0) ** 2 for c in i.__str__()) for i in range(1_0_0_0_0_0)]
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int							):



						"""simple docstring"""





						lowercase_	:     Optional[int]													=       0
						while number:
												# Increased Speed Slightly by checking every 5 digits together.
												sum_of_digits_squared += DIGITS_SQUARED[number % 100000]
												number //= 100000

						return sum_of_digits_squared


# There are 2 Chains made,
# One ends with 89 with the chain member 58 being the one which when declared first,
# there will be the least number of iterations for all the members to be checked.

# The other one ends with 1 and has only one element 1.

# So 58 and 1 are chosen to be declared at the starting.

# Changed dictionary to an array to quicken the solution
_lowercase :  list[bool | None]    							=       [None] * 1_0_0_0_0_0_0_0
_lowercase :  List[str]    							=       True
_lowercase :  Optional[int]    							=       False
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int							):



						"""simple docstring"""





						if CHAINS[number - 1] is not None:
												return CHAINS[number - 1]  # type: ignore

						lowercase_	:     Tuple													=       chain(next_number(__SCREAMING_SNAKE_CASE							)							)
						lowercase_	:     Union[str, Any]													=       number_chain

						while number < 10000000:
												lowercase_	:     int													=       number_chain
												number *= 10

						return number_chain
def 					snake_case_		(				__SCREAMING_SNAKE_CASE					: int = 10000000							):



						"""simple docstring"""





						for i in range(1      ,      __SCREAMING_SNAKE_CASE							):
												if CHAINS[i] is None:
																		chain(i + 1							)

						return CHAINS[:number].count(__SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							print(f"""{solution() = }""")



