


"""simple docstring"""






import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase    :     Any   						=		logging.get_logger(__name__)


UpperCAmelCase    :     List[Any]   						=		{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

UpperCAmelCase    :     List[str]   						=		{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

UpperCAmelCase    :     Tuple   						=		{'facebook/blenderbot_small-90M': 512}
def      lowerCamelCase       (     _UpperCamelCase		:      Dict							)		->       Optional[Any]:

							'''simple docstring'''



							__UpperCAmelCase   :						Optional[int]      							=					set()
							__UpperCAmelCase   :						Optional[int]      							=					word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														__UpperCAmelCase   :						Dict      							=					char

							__UpperCAmelCase   :						str      							=					set(_UpperCamelCase							)
							return pairs





class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					__a	       =							VOCAB_FILES_NAMES
					__a	       =							PRETRAINED_VOCAB_FILES_MAP
					__a	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__a	       =							["""input_ids""", """attention_mask"""]






					def __init__(       self    : Tuple ,       UpperCamelCase    : Optional[Any] ,       UpperCamelCase    : Any ,       UpperCamelCase    : Tuple="__start__" ,       UpperCamelCase    : Any="__end__" ,       UpperCamelCase    : List[Any]="__unk__" ,       UpperCamelCase    : Dict="__null__" ,       **UpperCamelCase    : str ,       ):
												'''simple docstring'''






												super().__init__(unk_token=UpperCamelCase ,       bos_token=UpperCamelCase ,       eos_token=UpperCamelCase ,       pad_token=UpperCamelCase ,       **UpperCamelCase  )

												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as vocab_handle:
																			__UpperCAmelCase   :						List[str]      							=					json.load(UpperCamelCase  )
												__UpperCAmelCase   :						List[Any]      							=					{v: k for k, v in self.encoder.items()}
												with open(UpperCamelCase ,       encoding="""utf-8"""  ) as merges_handle:
																			__UpperCAmelCase   :						Optional[int]      							=					merges_handle.read().split("""\n"""  )[1:-1]
												__UpperCAmelCase   :						Optional[int]      							=					[tuple(merge.split()  ) for merge in merges]
												__UpperCAmelCase   :						Tuple      							=					dict(zip(UpperCamelCase ,       range(len(UpperCamelCase  )  )  )  )
												__UpperCAmelCase   :						Optional[int]      							=					{}






					@property
					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												return len(self.encoder  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												return dict(self.encoder ,       **self.added_tokens_encoder  )






					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												if token in self.cache:
																			return self.cache[token]
												__UpperCAmelCase   :						List[str]      							=					re.sub("""([.,!?()])""" ,       R""" \1""" ,       UpperCamelCase  )
												__UpperCAmelCase   :						Union[str, Any]      							=					re.sub("""(')""" ,       R""" \1 """ ,       UpperCamelCase  )
												__UpperCAmelCase   :						List[str]      							=					re.sub(R"""\s{2,}""" ,       """ """ ,       UpperCamelCase  )
												if "\n" in token:
																			__UpperCAmelCase   :						Optional[Any]      							=					token.replace("""\n""" ,       """ __newln__"""  )

												__UpperCAmelCase   :						Union[str, Any]      							=					token.split(""" """  )
												__UpperCAmelCase   :						List[Any]      							=					[]
												for token in tokens:
																			if not len(UpperCamelCase  ):
																										continue

																			__UpperCAmelCase   :						str      							=					token.lower()
																			__UpperCAmelCase   :						str      							=					tuple(UpperCamelCase  )
																			__UpperCAmelCase   :						List[Any]      							=					tuple(list(word[:-1]  ) + [word[-1] + """</w>"""]  )
																			__UpperCAmelCase   :						Dict      							=					get_pairs(UpperCamelCase  )

																			if not pairs:
																										words.append(UpperCamelCase  )
																										continue

																			while True:
																										__UpperCAmelCase   :						int      							=					min(UpperCamelCase ,       key=lambda UpperCamelCase  : self.bpe_ranks.get(UpperCamelCase ,       float("""inf"""  )  )  )
																										if bigram not in self.bpe_ranks:
																																	break
																										__UpperCAmelCase							,__UpperCAmelCase   :						int      							=					bigram
																										__UpperCAmelCase   :						List[str]      							=					[]
																										__UpperCAmelCase   :						Dict      							=					0

																										while i < len(UpperCamelCase  ):
																																	try:
																																								__UpperCAmelCase   :						Tuple      							=					word.index(UpperCamelCase ,       UpperCamelCase  )
																																								new_word.extend(word[i:j]  )
																																								__UpperCAmelCase   :						Optional[Any]      							=					j
																																	except ValueError:
																																								new_word.extend(word[i:]  )
																																								break

																																	if word[i] == first and i < len(UpperCamelCase  ) - 1 and word[i + 1] == second:
																																								new_word.append(first + second  )
																																								i += 2
																																	else:
																																								new_word.append(word[i]  )
																																								i += 1
																										__UpperCAmelCase   :						Optional[int]      							=					tuple(UpperCamelCase  )
																										__UpperCAmelCase   :						int      							=					new_word
																										if len(UpperCamelCase  ) == 1:
																																	break
																										else:
																																	__UpperCAmelCase   :						Optional[int]      							=					get_pairs(UpperCamelCase  )
																			__UpperCAmelCase   :						Union[str, Any]      							=					"""@@ """.join(UpperCamelCase  )
																			__UpperCAmelCase   :						List[str]      							=					word[:-4]

																			__UpperCAmelCase   :						List[Any]      							=					word
																			words.append(UpperCamelCase  )
												return " ".join(UpperCamelCase  )






					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					[]

												__UpperCAmelCase   :						List[Any]      							=					re.findall(R"""\S+\n?""" ,       UpperCamelCase  )

												for token in words:
																			split_tokens.extend(list(self.bpe(UpperCamelCase  ).split(""" """  )  )  )
												return split_tokens






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					token.lower()
												return self.encoder.get(UpperCamelCase ,       self.encoder.get(self.unk_token  )  )






					def lowerCamelCase__							(       self    : Optional[int] ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												return self.decoder.get(UpperCamelCase ,       self.unk_token  )






					def lowerCamelCase__							(       self    : Dict ,       UpperCamelCase    : List[str]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					""" """.join(UpperCamelCase  ).replace("""@@ """ ,       """"""  ).strip()
												return out_string






					def lowerCamelCase__							(       self    : List[Any] ,       UpperCamelCase    : str ,       UpperCamelCase    : Optional[str] = None  ):
												'''simple docstring'''






												if not os.path.isdir(UpperCamelCase  ):
																			logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
																			return
												__UpperCAmelCase   :						Dict      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
												__UpperCAmelCase   :						List[Any]      							=					os.path.join(
												    UpperCamelCase ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as f:
																			f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=UpperCamelCase ,       ensure_ascii=UpperCamelCase  ) + """\n"""  )

												__UpperCAmelCase   :						Any      							=					0
												with open(UpperCamelCase ,       """w""" ,       encoding="""utf-8"""  ) as writer:
																			writer.write("""#version: 0.2\n"""  )
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda UpperCamelCase  : kv[1]  ):
																										if index != token_index:
																																	logger.warning(
																																	    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																																	    """ Please check that the tokenizer is not corrupted!"""  )
																																	__UpperCAmelCase   :						Optional[Any]      							=					token_index
																										writer.write(""" """.join(UpperCamelCase  ) + """\n"""  )
																										index += 1

												return vocab_file, merge_file




"""simple docstring"""






def      lowerCamelCase       (     _UpperCamelCase		:      float       ,			_UpperCamelCase		:      float       ,			_UpperCamelCase		:      int							)		->       float:

							'''simple docstring'''



							if principal <= 0:
														raise Exception("""Principal borrowed must be > 0"""							)
							if rate_per_annum < 0:
														raise Exception("""Rate of interest must be >= 0"""							)
							if years_to_repay <= 0 or not isinstance(_UpperCamelCase       ,			_UpperCamelCase							):
														raise Exception("""Years to repay must be an integer > 0"""							)

							# Yearly rate is divided by 12 to get monthly rate
							__UpperCAmelCase   :						List[Any]      							=					rate_per_annum / 1_2

							# Years to repay is multiplied by 12 to get number of payments as payment is monthly
							__UpperCAmelCase   :						Optional[int]      							=					years_to_repay * 1_2

							return (
							    principal
							    * rate_per_month
							    * (1 + rate_per_month) ** number_of_payments
							    / ((1 + rate_per_month) ** number_of_payments - 1)
							)


if __name__ == "__main__":
							import doctest

							doctest.testmod()
