


"""simple docstring"""






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase    :     Optional[Any]   						=		logging.get_logger(__name__)

UpperCAmelCase    :     str   						=		{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCAmelCase    :     Optional[Any]   						=		{
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

UpperCAmelCase    :     Tuple   						=		{
    'ctrl': 256,
}

UpperCAmelCase    :     Optional[Any]   						=		{
    'Pregnancy': 16_8629,
    'Christianity': 7675,
    'Explain': 10_6423,
    'Fitness': 6_3440,
    'Saving': 6_3163,
    'Ask': 2_7171,
    'Ass': 9_5985,
    'Joke': 16_3509,
    'Questions': 4_5622,
    'Thoughts': 4_9605,
    'Retail': 5_2342,
    'Feminism': 16_4338,
    'Writing': 1_1992,
    'Atheism': 19_2263,
    'Netflix': 4_8616,
    'Computing': 3_9639,
    'Opinion': 4_3213,
    'Alone': 4_4967,
    'Funny': 5_8917,
    'Gaming': 4_0358,
    'Human': 4088,
    'India': 1331,
    'Joker': 7_7138,
    'Diet': 3_6206,
    'Legal': 1_1859,
    'Norman': 4939,
    'Tip': 7_2689,
    'Weight': 5_2343,
    'Movies': 4_6273,
    'Running': 2_3425,
    'Science': 2090,
    'Horror': 3_7793,
    'Confession': 6_0572,
    'Finance': 1_2250,
    'Politics': 1_6360,
    'Scary': 19_1985,
    'Support': 1_2654,
    'Technologies': 3_2516,
    'Teenage': 6_6160,
    'Event': 3_2769,
    'Learned': 6_7460,
    'Notion': 18_2770,
    'Wikipedia': 3_7583,
    'Books': 6665,
    'Extract': 7_6050,
    'Confessions': 10_2701,
    'Conspiracy': 7_5932,
    'Links': 6_3674,
    'Narcissus': 15_0425,
    'Relationship': 5_4766,
    'Relationships': 13_4796,
    'Reviews': 4_1671,
    'News': 4256,
    'Translation': 2_6820,
    'multilingual': 12_8406,
}
def      lowerCamelCase       (     _UpperCamelCase		:      Optional[int]							)		->       List[str]:

							'''simple docstring'''



							__UpperCAmelCase   :						List[str]      							=					set()
							__UpperCAmelCase   :						Tuple      							=					word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														__UpperCAmelCase   :						str      							=					char

							__UpperCAmelCase   :						Optional[Any]      							=					set(_UpperCamelCase							)
							return pairs





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							CONTROL_CODES






					def __init__(       self    : List[Any] ,       UpperCamelCase    : Optional[Any] ,       UpperCamelCase    : Optional[int] ,       UpperCamelCase    : Tuple="<unk>" ,       **UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												super().__init__(unk_token=UpperCamelCase ,       **UpperCamelCase  )

												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as vocab_handle:
																			__UpperCAmelCase   :						Tuple      							=					json.load(UpperCamelCase  )
												__UpperCAmelCase   :						Tuple      							=					{v: k for k, v in self.encoder.items()}
												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as merges_handle:
																			__UpperCAmelCase   :						Tuple      							=					merges_handle.read().split("""\n"""  )[1:-1]
												__UpperCAmelCase   :						Union[str, Any]      							=					[tuple(merge.split()  ) for merge in merges]
												__UpperCAmelCase   :						str      							=					dict(zip(UpperCamelCase ,       range(len(UpperCamelCase  )  )  )  )
												__UpperCAmelCase   :						Optional[Any]      							=					{}






					@property
					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												return len(self.encoder  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												return dict(self.encoder ,       **self.added_tokens_encoder  )






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												if token in self.cache:
																			return self.cache[token]
												__UpperCAmelCase   :						Optional[Any]      							=					tuple(UpperCamelCase  )
												__UpperCAmelCase   :						str      							=					tuple(list(word[:-1]  ) + [word[-1] + """</w>"""]  )
												__UpperCAmelCase   :						Dict      							=					get_pairs(UpperCamelCase  )

												if not pairs:
																			return token

												while True:
																			__UpperCAmelCase   :						List[Any]      							=					min(UpperCamelCase ,       key=lambda UpperCamelCase  : self.bpe_ranks.get(UpperCamelCase ,       float("""inf"""  )  )  )
																			if bigram not in self.bpe_ranks:
																										break
																			__UpperCAmelCase							,__UpperCAmelCase   :						List[str]      							=					bigram
																			__UpperCAmelCase   :						Dict      							=					[]
																			__UpperCAmelCase   :						Dict      							=					0
																			while i < len(UpperCamelCase  ):
																										try:
																																	__UpperCAmelCase   :						str      							=					word.index(UpperCamelCase ,       UpperCamelCase  )
																										except ValueError:
																																	new_word.extend(word[i:]  )
																																	break
																										else:
																																	new_word.extend(word[i:j]  )
																																	__UpperCAmelCase   :						Optional[int]      							=					j

																										if word[i] == first and i < len(UpperCamelCase  ) - 1 and word[i + 1] == second:
																																	new_word.append(first + second  )
																																	i += 2
																										else:
																																	new_word.append(word[i]  )
																																	i += 1
																			__UpperCAmelCase   :						Any      							=					tuple(UpperCamelCase  )
																			__UpperCAmelCase   :						Dict      							=					new_word
																			if len(UpperCamelCase  ) == 1:
																										break
																			else:
																										__UpperCAmelCase   :						str      							=					get_pairs(UpperCamelCase  )
												__UpperCAmelCase   :						Tuple      							=					"""@@ """.join(UpperCamelCase  )
												__UpperCAmelCase   :						int      							=					word[:-4]
												__UpperCAmelCase   :						int      							=					word
												return word






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					[]

												__UpperCAmelCase   :						Dict      							=					re.findall(R"""\S+\n?""" ,       UpperCamelCase  )

												for token in words:
																			split_tokens.extend(list(self.bpe(UpperCamelCase  ).split(""" """  )  )  )
												return split_tokens






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : Optional[int]  ):
												'''simple docstring'''






												return self.encoder.get(UpperCamelCase ,       self.encoder.get(self.unk_token  )  )






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : Optional[Any]  ):
												'''simple docstring'''






												return self.decoder.get(UpperCamelCase ,       self.unk_token  )






					def lowerCamelCase__							(       self    : Optional[Any] ,       UpperCamelCase    : List[str]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					""" """.join(UpperCamelCase  ).replace("""@@ """ ,       """"""  ).strip()
												return out_string






					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Optional[Any]      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
												__UpperCAmelCase   :						List[Any]      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as f:
																			f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=UpperCamelCase ,       ensure_ascii=UpperCamelCase  ) + """\n"""  )

												__UpperCAmelCase   :						Optional[int]      							=					0
												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as writer:
																			writer.write("""#version: 0.2\n"""  )
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda UpperCamelCase  : kv[1]  ):
																										if index != token_index:
																																	logger.warning(
																																	    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																	    """ Please check that the tokenizer is not corrupted!"""  )
																																	__UpperCAmelCase   :						Union[str, Any]      							=					token_index
																										writer.write(""" """.join(UpperCamelCase  ) + """\n"""  )
																										index += 1

												return vocab_file, merge_file

					# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
					#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
					#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
					#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
					#     return ''.join(tokens_generated_so_far)




"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
							import PIL


UpperCAmelCase    :     Optional[Any]   						=		logging.get_logger(__name__)
def      lowerCamelCase       (     _UpperCamelCase		:      Any       ,			_UpperCamelCase		:      Optional[Any]							)		->       int:

							'''simple docstring'''



							__UpperCAmelCase   :						int      							=					b.T
							__UpperCAmelCase   :						Union[str, Any]      							=					np.sum(np.square(_UpperCamelCase							)       ,			axis=1							)
							__UpperCAmelCase   :						List[Any]      							=					np.sum(np.square(_UpperCamelCase							)       ,			axis=0							)
							__UpperCAmelCase   :						int      							=					np.matmul(_UpperCamelCase       ,			_UpperCamelCase							)
							__UpperCAmelCase   :						Tuple      							=					aa[:, None] - 2 * ab + ba[None, :]
							return d
def      lowerCamelCase       (     _UpperCamelCase		:      Tuple       ,			_UpperCamelCase		:      Optional[Any]							)		->       Optional[int]:

							'''simple docstring'''



							__UpperCAmelCase   :						Tuple      							=					x.reshape(-1       ,			3							)
							__UpperCAmelCase   :						Optional[Any]      							=					squared_euclidean_distance(_UpperCamelCase       ,			_UpperCamelCase							)
							return np.argmin(_UpperCamelCase       ,			axis=1							)





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							["""pixel_values"""]






					def __init__(       self    : List[str] ,       UpperCamelCase    : Optional[Union[List[List[int]], np.ndarray]] = None ,       UpperCamelCase    : bool = True ,       UpperCamelCase    : Dict[str, int] = None ,       UpperCamelCase    : PILImageResampling = PILImageResampling.BILINEAR ,       UpperCamelCase    : bool = True ,       UpperCamelCase    : bool = True ,       **UpperCamelCase    : int ,       ):
												'''simple docstring'''






												super().__init__(**UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					size if size is not None else {"""height""": 256, """width""": 256}
												__UpperCAmelCase   :						Tuple      							=					get_size_dict(UpperCamelCase  )
												__UpperCAmelCase   :						Union[str, Any]      							=					np.array(UpperCamelCase  ) if clusters is not None else None
												__UpperCAmelCase   :						int      							=					do_resize
												__UpperCAmelCase   :						List[str]      							=					size
												__UpperCAmelCase   :						Dict      							=					resample
												__UpperCAmelCase   :						int      							=					do_normalize
												__UpperCAmelCase   :						List[Any]      							=					do_color_quantize






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : np.ndarray ,       UpperCamelCase    : Dict[str, int] ,       UpperCamelCase    : PILImageResampling = PILImageResampling.BILINEAR ,       UpperCamelCase    : Optional[Union[str, ChannelDimension]] = None ,       **UpperCamelCase    : str ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					get_size_dict(UpperCamelCase  )
												if "height" not in size or "width" not in size:
																			raise ValueError(f'''Size dictionary must contain both height and width keys. Got {size.keys()}'''  )
												return resize(
												    UpperCamelCase ,       size=(size["""height"""], size["""width"""]) ,       resample=UpperCamelCase ,       data_format=UpperCamelCase ,       **UpperCamelCase  )






					def lowerCamelCase__							(       self    : Dict ,       UpperCamelCase    : np.ndarray ,       UpperCamelCase    : Optional[Union[str, ChannelDimension]] = None ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[Any]      							=					rescale(image=UpperCamelCase ,       scale=1 / 127.5 ,       data_format=UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					image - 1
												return image






					def lowerCamelCase__							(       self    : Dict ,       UpperCamelCase    : ImageInput ,       UpperCamelCase    : bool = None ,       UpperCamelCase    : Dict[str, int] = None ,       UpperCamelCase    : PILImageResampling = None ,       UpperCamelCase    : bool = None ,       UpperCamelCase    : Optional[bool] = None ,       UpperCamelCase    : Optional[Union[List[List[int]], np.ndarray]] = None ,       UpperCamelCase    : Optional[Union[str, TensorType]] = None ,       UpperCamelCase    : Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST ,       **UpperCamelCase    : Optional[int] ,       ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					do_resize if do_resize is not None else self.do_resize
												__UpperCAmelCase   :						Any      							=					size if size is not None else self.size
												__UpperCAmelCase   :						Any      							=					get_size_dict(UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					resample if resample is not None else self.resample
												__UpperCAmelCase   :						Any      							=					do_normalize if do_normalize is not None else self.do_normalize
												__UpperCAmelCase   :						Any      							=					do_color_quantize if do_color_quantize is not None else self.do_color_quantize
												__UpperCAmelCase   :						Tuple      							=					clusters if clusters is not None else self.clusters
												__UpperCAmelCase   :						Tuple      							=					np.array(UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					make_list_of_images(UpperCamelCase  )

												if not valid_images(UpperCamelCase  ):
																			raise ValueError(
																			    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
																			    """torch.Tensor, tf.Tensor or jax.ndarray."""  )

												if do_resize and size is None or resample is None:
																			raise ValueError("""Size and resample must be specified if do_resize is True."""  )

												if do_color_quantize and clusters is None:
																			raise ValueError("""Clusters must be specified if do_color_quantize is True."""  )

												# All transformations expect numpy arrays.
												__UpperCAmelCase   :						Tuple      							=					[to_numpy_array(UpperCamelCase  ) for image in images]

												if do_resize:
																			__UpperCAmelCase   :						Dict      							=					[self.resize(image=UpperCamelCase ,       size=UpperCamelCase ,       resample=UpperCamelCase  ) for image in images]

												if do_normalize:
																			__UpperCAmelCase   :						Any      							=					[self.normalize(image=UpperCamelCase  ) for image in images]

												if do_color_quantize:
																			__UpperCAmelCase   :						str      							=					[to_channel_dimension_format(UpperCamelCase ,       ChannelDimension.LAST  ) for image in images]
																			# color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
																			__UpperCAmelCase   :						Optional[int]      							=					np.array(UpperCamelCase  )
																			__UpperCAmelCase   :						Any      							=					color_quantize(UpperCamelCase ,       UpperCamelCase  ).reshape(images.shape[:-1]  )

																			# flatten to (batch_size, height*width)
																			__UpperCAmelCase   :						Dict      							=					images.shape[0]
																			__UpperCAmelCase   :						Optional[Any]      							=					images.reshape(UpperCamelCase ,       -1  )

																			# We need to convert back to a list of images to keep consistent behaviour across processors.
																			__UpperCAmelCase   :						int      							=					list(UpperCamelCase  )
												else:
																			__UpperCAmelCase   :						List[Any]      							=					[to_channel_dimension_format(UpperCamelCase ,       UpperCamelCase  ) for image in images]

												__UpperCAmelCase   :						List[str]      							=					{"""input_ids""": images}
												return BatchFeature(data=UpperCamelCase ,       tensor_type=UpperCamelCase  )
