




"""simple docstring"""




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase							:			Optional[int]     							=							logging.get_logger(__name__)

lowerCAmelCase							:			Union[str, Any]     							=							{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCAmelCase							:			List[Any]     							=							{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCAmelCase							:			Tuple     							=							{
    """ctrl""": 256,
}

lowerCAmelCase							:			Dict     							=							{
    """Pregnancy""": 16_8629,
    """Christianity""": 7675,
    """Explain""": 10_6423,
    """Fitness""": 6_3440,
    """Saving""": 6_3163,
    """Ask""": 2_7171,
    """Ass""": 9_5985,
    """Joke""": 16_3509,
    """Questions""": 4_5622,
    """Thoughts""": 4_9605,
    """Retail""": 5_2342,
    """Feminism""": 16_4338,
    """Writing""": 1_1992,
    """Atheism""": 19_2263,
    """Netflix""": 4_8616,
    """Computing""": 3_9639,
    """Opinion""": 4_3213,
    """Alone""": 4_4967,
    """Funny""": 5_8917,
    """Gaming""": 4_0358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 7_7138,
    """Diet""": 3_6206,
    """Legal""": 1_1859,
    """Norman""": 4939,
    """Tip""": 7_2689,
    """Weight""": 5_2343,
    """Movies""": 4_6273,
    """Running""": 2_3425,
    """Science""": 2090,
    """Horror""": 3_7793,
    """Confession""": 6_0572,
    """Finance""": 1_2250,
    """Politics""": 1_6360,
    """Scary""": 19_1985,
    """Support""": 1_2654,
    """Technologies""": 3_2516,
    """Teenage""": 6_6160,
    """Event""": 3_2769,
    """Learned""": 6_7460,
    """Notion""": 18_2770,
    """Wikipedia""": 3_7583,
    """Books""": 6665,
    """Extract""": 7_6050,
    """Confessions""": 10_2701,
    """Conspiracy""": 7_5932,
    """Links""": 6_3674,
    """Narcissus""": 15_0425,
    """Relationship""": 5_4766,
    """Relationships""": 13_4796,
    """Reviews""": 4_1671,
    """News""": 4256,
    """Translation""": 2_6820,
    """multilingual""": 12_8406,
}



def     a__ (				snake_case__				)						->					int:
			lowerCamelCase			      =					set()
			lowerCamelCase			      =					word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)				)
						lowerCamelCase			      =					char

			lowerCamelCase			      =					set(snake_case__				)
			return pairs






class     __magic_name__							(       UpperCAmelCase__				):






				'''simple docstring'''
				__UpperCamelCase			  =						VOCAB_FILES_NAMES
				__UpperCamelCase			  =						PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase			  =						CONTROL_CODES





				def __init__(       self		, _a		, _a		, _a="<unk>"		, **_a ):





							"""simple docstring"""
							super().__init__(unk_token=_a		, **_a )

							with open(_a		, encoding="""utf-8""" ) as vocab_handle:
										lowerCamelCase			      =					json.load(_a )
							lowerCamelCase			      =					{v: k for k, v in self.encoder.items()}
							with open(_a		, encoding="""utf-8""" ) as merges_handle:
										lowerCamelCase			      =					merges_handle.read().split("""\n""" )[1:-1]
							lowerCamelCase			      =					[tuple(merge.split() ) for merge in merges]
							lowerCamelCase			      =					dict(zip(_a		, range(len(_a ) ) ) )
							lowerCamelCase			      =					{}





				@property
				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							return len(self.encoder )





				def    _lowerCAmelCase				(       self ):





							"""simple docstring"""
							return dict(self.encoder		, **self.added_tokens_encoder )





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							if token in self.cache:
										return self.cache[token]
							lowerCamelCase			      =					tuple(_a )
							lowerCamelCase			      =					tuple(list(word[:-1] ) + [word[-1] + """</w>"""] )
							lowerCamelCase			      =					get_pairs(_a )

							if not pairs:
										return token

							while True:
										lowerCamelCase			      =					min(_a		, key=lambda _a : self.bpe_ranks.get(_a		, float("""inf""" ) ) )
										if bigram not in self.bpe_ranks:
													break
										lowerCamelCase      ,   lowerCamelCase			      =					bigram
										lowerCamelCase			      =					[]
										lowerCamelCase			      =					0
										while i < len(_a ):
													try:
																lowerCamelCase			      =					word.index(_a		, _a )
													except ValueError:
																new_word.extend(word[i:] )
																break
													else:
																new_word.extend(word[i:j] )
																lowerCamelCase			      =					j

													if word[i] == first and i < len(_a ) - 1 and word[i + 1] == second:
																new_word.append(first + second )
																i += 2
													else:
																new_word.append(word[i] )
																i += 1
										lowerCamelCase			      =					tuple(_a )
										lowerCamelCase			      =					new_word
										if len(_a ) == 1:
													break
										else:
													lowerCamelCase			      =					get_pairs(_a )
							lowerCamelCase			      =					"""@@ """.join(_a )
							lowerCamelCase			      =					word[:-4]
							lowerCamelCase			      =					word
							return word





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					[]

							lowerCamelCase			      =					re.findall(r"""\S+\n?"""		, _a )

							for token in words:
										split_tokens.extend(list(self.bpe(_a ).split(""" """ ) ) )
							return split_tokens





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							return self.encoder.get(_a		, self.encoder.get(self.unk_token ) )





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							return self.decoder.get(_a		, self.unk_token )





				def    _lowerCAmelCase				(       self		, _a ):





							"""simple docstring"""
							lowerCamelCase			      =					""" """.join(_a ).replace("""@@ """		, """""" ).strip()
							return out_string





				def    _lowerCAmelCase				(       self		, _a		, _a = None ):





							"""simple docstring"""
							if not os.path.isdir(_a ):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
										return
							lowerCamelCase			      =					os.path.join(
							    _a		, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
							lowerCamelCase			      =					os.path.join(
							    _a		, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

							with open(_a		, """w"""		, encoding="""utf-8""" ) as f:
										f.write(json.dumps(self.encoder		, indent=2		, sort_keys=_a		, ensure_ascii=_a ) + """\n""" )

							lowerCamelCase			      =					0
							with open(_a		, """w"""		, encoding="""utf-8""" ) as writer:
										writer.write("""#version: 0.2\n""" )
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		, key=lambda _a : kv[1] ):
													if index != token_index:
																logger.warning(
																    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																    """ Please check that the tokenizer is not corrupted!""" )
																lowerCamelCase			      =					token_index
													writer.write(""" """.join(_a ) + """\n""" )
													index += 1

							return vocab_file, merge_file

				# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
				#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
				#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
				#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
				#     return ''.join(tokens_generated_so_far)









"""simple docstring"""




def     a__ (				snake_case__				)						->					bool:
			lowerCamelCase			      =					(1 + 24 * n) ** 0.5
			return ((1 + root) / 6) % 1 == 0



def     a__ (				snake_case__ = 50_00				)						->					int:
			lowerCamelCase			      =					[(i * (3 * i - 1)) // 2 for i in range(1       , snake_case__				)]
			for i, pentagonal_i in enumerate(snake_case__				):
						for j in range(snake_case__       , len(snake_case__				)				):
									lowerCamelCase			      =					pentagonal_nums[j]
									lowerCamelCase			      =					pentagonal_i + pentagonal_j
									lowerCamelCase			      =					pentagonal_j - pentagonal_i
									if is_pentagonal(snake_case__				) and is_pentagonal(snake_case__				):
												return b

			return -1


if __name__ == "__main__":
					print(F"""{solution() = }""")



