





"""simple docstring"""
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast






class 					_UpperCAmelCase   (	unittest.TestCase ):


					def 	lowerCamelCase       (					self			:List[Any]		):
												# A mock response for an HTTP head request to emulate server down
												A		=  mock.Mock()
												A		=  5_00
												A		=  {}
												A		=  HTTPError
												A		=  {}

												# Download this model to make sure it's in the cache.
												A		=  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("requests.Session.request"     ,       return_value=__UpperCamelCase		) as mock_head:
																			A		=  BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"		)
																			# This check we did call the fake head request
																			mock_head.assert_called()


					@require_tokenizers
					def 	lowerCamelCase       (					self			:Optional[Any]		):
												# A mock response for an HTTP head request to emulate server down
												A		=  mock.Mock()
												A		=  5_00
												A		=  {}
												A		=  HTTPError
												A		=  {}

												# Download this model to make sure it's in the cache.
												A		=  GPTaTokenizerFast.from_pretrained("gpt2"		)

												# Under the mock environment we get a 500 error when trying to reach the tokenizer.
												with mock.patch("requests.Session.request"     ,       return_value=__UpperCamelCase		) as mock_head:
																			A		=  GPTaTokenizerFast.from_pretrained("gpt2"		)
																			# This check we did call the fake head request
																			mock_head.assert_called()


					def 	lowerCamelCase       (					self			:Tuple		):
												# This test is for deprecated behavior and can be removed in v5
												try:
																			A		=  tempfile.mktemp()
																			with open(__UpperCamelCase     ,       "wb"		) as f:
																										http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"     ,       __UpperCamelCase		)

																			A		=  AlbertTokenizer.from_pretrained(__UpperCamelCase		)
												finally:
																			os.remove(__UpperCamelCase		)

												# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
												# the current folder and have the right name.
												if os.path.isfile("tokenizer.json"		):
																			# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
																			return
												try:
																			with open("tokenizer.json"     ,       "wb"		) as f:
																										http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"     ,       __UpperCamelCase		)
																			A		=  AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"		)
																			# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
																			self.assertEqual(tokenizer.vocab_size     ,       10_00		)
																			# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

												finally:
																			os.remove("tokenizer.json"		)


					def 	lowerCamelCase       (					self			:Union[str, Any]		):
												# This test is for deprecated behavior and can be removed in v5
												A		=  AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		)







@is_staging_test
class 					_UpperCAmelCase   (	unittest.TestCase ):
					UpperCamelCase      =							['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']


					@classmethod
					def 	lowerCamelCase       (					cls			:Tuple		):
												A		=  TOKEN
												HfFolder.save_token(__UpperCamelCase		)


					@classmethod
					def 	lowerCamelCase       (					cls			:Any		):
												try:
																			delete_repo(token=cls._token     ,       repo_id="test-tokenizer"		)
												except HTTPError:
																			pass

												try:
																			delete_repo(token=cls._token     ,       repo_id="valid_org/test-tokenizer-org"		)
												except HTTPError:
																			pass

												try:
																			delete_repo(token=cls._token     ,       repo_id="test-dynamic-tokenizer"		)
												except HTTPError:
																			pass


					def 	lowerCamelCase       (					self			:int		):
												with tempfile.TemporaryDirectory() as tmp_dir:
																			A		=  os.path.join(__UpperCamelCase     ,       "vocab.txt"		)
																			with open(__UpperCamelCase     ,       "w"     ,       encoding="utf-8"		) as vocab_writer:
																										vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
																			A		=  BertTokenizer(__UpperCamelCase		)

												tokenizer.push_to_hub("test-tokenizer"     ,       use_auth_token=self._token		)
												A		=  BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"		)
												self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab		)

												# Reset repo
												delete_repo(token=self._token     ,       repo_id="test-tokenizer"		)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(__UpperCamelCase     ,       repo_id="test-tokenizer"     ,       push_to_hub=__UpperCamelCase     ,       use_auth_token=self._token		)

												A		=  BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"		)
												self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab		)


					def 	lowerCamelCase       (					self			:Optional[int]		):
												with tempfile.TemporaryDirectory() as tmp_dir:
																			A		=  os.path.join(__UpperCamelCase     ,       "vocab.txt"		)
																			with open(__UpperCamelCase     ,       "w"     ,       encoding="utf-8"		) as vocab_writer:
																										vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
																			A		=  BertTokenizer(__UpperCamelCase		)

												tokenizer.push_to_hub("valid_org/test-tokenizer-org"     ,       use_auth_token=self._token		)
												A		=  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
												self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab		)

												# Reset repo
												delete_repo(token=self._token     ,       repo_id="valid_org/test-tokenizer-org"		)

												# Push to hub via save_pretrained
												with tempfile.TemporaryDirectory() as tmp_dir:
																			tokenizer.save_pretrained(
																			    __UpperCamelCase     ,       repo_id="valid_org/test-tokenizer-org"     ,       push_to_hub=__UpperCamelCase     ,       use_auth_token=self._token		)

												A		=  BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"		)
												self.assertDictEqual(new_tokenizer.vocab     ,       tokenizer.vocab		)


					@require_tokenizers
					def 	lowerCamelCase       (					self			:str		):
												CustomTokenizer.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																			A		=  os.path.join(__UpperCamelCase     ,       "vocab.txt"		)
																			with open(__UpperCamelCase     ,       "w"     ,       encoding="utf-8"		) as vocab_writer:
																										vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)
																			A		=  CustomTokenizer(__UpperCamelCase		)

												# No fast custom tokenizer
												tokenizer.push_to_hub("test-dynamic-tokenizer"     ,       use_auth_token=self._token		)

												A		=  AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,       trust_remote_code=__UpperCamelCase		)
												# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__     ,       "CustomTokenizer"		)

												# Fast and slow custom tokenizer
												CustomTokenizerFast.register_for_auto_class()
												with tempfile.TemporaryDirectory() as tmp_dir:
																			A		=  os.path.join(__UpperCamelCase     ,       "vocab.txt"		)
																			with open(__UpperCamelCase     ,       "w"     ,       encoding="utf-8"		) as vocab_writer:
																										vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]		)		)

																			A		=  BertTokenizerFast.from_pretrained(__UpperCamelCase		)
																			bert_tokenizer.save_pretrained(__UpperCamelCase		)
																			A		=  CustomTokenizerFast.from_pretrained(__UpperCamelCase		)

												tokenizer.push_to_hub("test-dynamic-tokenizer"     ,       use_auth_token=self._token		)

												A		=  AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,       trust_remote_code=__UpperCamelCase		)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__     ,       "CustomTokenizerFast"		)
												A		=  AutoTokenizer.from_pretrained(
												    f"{USER}/test-dynamic-tokenizer"     ,       use_fast=__UpperCamelCase     ,       trust_remote_code=__UpperCamelCase		)
												# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
												self.assertEqual(tokenizer.__class__.__name__     ,       "CustomTokenizer"		)







class 					_UpperCAmelCase   (	unittest.TestCase ):


					def 	lowerCamelCase       (					self			:str		):
												A		=  Trie()
												trie.add("Hello 友達"		)
												self.assertEqual(trie.data     ,       {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}		)
												trie.add("Hello"		)
												trie.data
												self.assertEqual(trie.data     ,       {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}		)


					def 	lowerCamelCase       (					self			:Dict		):
												A		=  Trie()
												self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)     ,       ["[CLS] This is a extra_id_100"]		)
												trie.add("[CLS]"		)
												trie.add("extra_id_1"		)
												trie.add("extra_id_100"		)
												self.assertEqual(trie.split("[CLS] This is a extra_id_100"		)     ,       ["[CLS]", " This is a ", "extra_id_100"]		)


					def 	lowerCamelCase       (					self			:str		):
												A		=  Trie()
												trie.add("A"		)
												self.assertEqual(trie.split("ABC"		)     ,       ["A", "BC"]		)
												self.assertEqual(trie.split("BCA"		)     ,       ["BC", "A"]		)


					def 	lowerCamelCase       (					self			:Any		):
												A		=  Trie()
												trie.add("TOKEN]"		)
												trie.add("[SPECIAL_TOKEN]"		)
												self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)     ,       ["This is something ", "[SPECIAL_TOKEN]"]		)


					def 	lowerCamelCase       (					self			:List[str]		):
												A		=  Trie()
												trie.add("A"		)
												trie.add("P"		)
												trie.add("[SPECIAL_TOKEN]"		)
												self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"		)     ,       ["This is something ", "[SPECIAL_TOKEN]"]		)


					def 	lowerCamelCase       (					self			:Tuple		):
												A		=  Trie()
												trie.add("AB"		)
												trie.add("B"		)
												trie.add("C"		)
												self.assertEqual(trie.split("ABC"		)     ,       ["AB", "C"]		)


					def 	lowerCamelCase       (					self			:str		):
												A		=  Trie()
												trie.add("ABC"		)
												trie.add("B"		)
												trie.add("CD"		)
												self.assertEqual(trie.split("ABCD"		)     ,       ["ABC", "D"]		)


					def 	lowerCamelCase       (					self			:Dict		):
												# Even if the offsets are wrong, we necessarily output correct string
												# parts.
												A		=  Trie()
												A		=  trie.cut_text("ABC"     ,       [0, 0, 2, 1, 2, 3]		)
												self.assertEqual(__UpperCamelCase     ,       ["AB", "C"]		)









"""simple docstring"""
def 					A__				(			UpperCamelCase       ):
							A		=  generate_pascal_triangle(UpperCamelCase       )
							for row_idx in range(UpperCamelCase       ):
														# Print left spaces
														for _ in range(num_rows - row_idx - 1       ):
																					print(end=" "       )
														# Print row values
														for col_idx in range(row_idx + 1       ):
																					if col_idx != row_idx:
																												print(triangle[row_idx][col_idx]					,  end=" "       )
																					else:
																												print(triangle[row_idx][col_idx]					,  end=""       )
														print()






def 					A__				(			UpperCamelCase       ):

							if not isinstance(UpperCamelCase					,  UpperCamelCase       ):
														raise TypeError("The input value of 'num_rows' should be 'int'"       )

							if num_rows == 0:
														return []
							elif num_rows < 0:
														raise ValueError(
														    "The input value of 'num_rows' should be greater than or equal to 0"       )

							A		=  []
							for current_row_idx in range(UpperCamelCase       ):
														A		=  populate_current_row(UpperCamelCase					,  UpperCamelCase       )
														triangle.append(UpperCamelCase       )
							return triangle






def 					A__				(			UpperCamelCase					,  UpperCamelCase       ):
							A		=  [-1] * (current_row_idx + 1)
							# first and last elements of current row are equal to 1
							A,	A		=  1, 1
							for current_col_idx in range(1					,  UpperCamelCase       ):
														calculate_current_element(
														    UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase       )
							return current_row






def 					A__				(			UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  UpperCamelCase					,  ):
							A		=  triangle[current_row_idx - 1][current_col_idx - 1]
							A		=  triangle[current_row_idx - 1][current_col_idx]
							A		=  above_to_left_elt + above_to_right_elt






def 					A__				(			UpperCamelCase       ):

							if not isinstance(UpperCamelCase					,  UpperCamelCase       ):
														raise TypeError("The input value of 'num_rows' should be 'int'"       )

							if num_rows == 0:
														return []
							elif num_rows < 0:
														raise ValueError(
														    "The input value of 'num_rows' should be greater than or equal to 0"       )

							A		=  [[1]]

							for row_index in range(1					,  UpperCamelCase       ):
														A		=  [0] + result[-1] + [0]
														A		=  row_index + 1
														# Calculate the number of distinct elements in a row
														A		=  sum(divmod(UpperCamelCase					,  2       )       )
														A		=  [
														    temp_row[i - 1] + temp_row[i] for i in range(1					,  distinct_elements + 1       )
														]
														A		=  row_first_half[: (row_index + 1) // 2]
														row_second_half.reverse()
														A		=  row_first_half + row_second_half
														result.append(UpperCamelCase       )

							return result






def 					A__				(			):
							from collections.abc import Callable
							from timeit import timeit

							def benchmark_a_function(UpperCamelCase					,  UpperCamelCase       ) -> None:
														A		=  F"{func.__name__}({value})"
														A		=  timeit(F"__main__.{call}"					,  setup="import __main__"       )
														# print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
														print(F"{call:38} -- {timing:.4f} seconds"       )

							for value in range(15       ):  # (1, 7, 14):
														for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
																					benchmark_a_function(UpperCamelCase					,  UpperCamelCase       )
														print()


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		benchmark()


