


"""simple docstring"""







import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  UpperCamelCase_	( UpperCamelCase			,  unittest.TestCase):

	"""simple docstring"""





	snake_case__				:  Union[str, Any]    					=			FunnelTokenizer
	snake_case__				:  List[str]    					=			FunnelTokenizerFast
	snake_case__				:  str    					=			True
	snake_case__				:  Optional[Any]    					=			True




	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	str:
						super().setUp()

						__SCREAMING_SNAKE_CASE						   =				[
						    "<unk>",
						    "<cls>",
						    "<sep>",
						    "want",
						    "##want",
						    "##ed",
						    "wa",
						    "un",
						    "runn",
						    "##ing",
						    ",",
						    "low",
						    "lowest",
						]
						__SCREAMING_SNAKE_CASE						   =				os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES["vocab_file"]   )
						with open(self.vocab_file			,			"w"			,			encoding="utf-8"   ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )




	def   UpperCAmelCase_      ( self			:				List[str]			,			**UpperCAmelCase__			:				str   )       ->	Any:
						return FunnelTokenizer.from_pretrained(self.tmpdirname			,			**UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				Dict			,			**UpperCAmelCase__			:				int   )       ->	List[str]:
						return FunnelTokenizerFast.from_pretrained(self.tmpdirname			,			**UpperCAmelCase__   )




	def   UpperCAmelCase_      ( self			:				List[str]			,			UpperCAmelCase__			:				int   )       ->	int:
						__SCREAMING_SNAKE_CASE						   =				"UNwant\u00E9d,running"
						__SCREAMING_SNAKE_CASE						   =				"unwanted, running"
						return input_text, output_text




	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	Dict:
						__SCREAMING_SNAKE_CASE						   =				self.tokenizer_class(self.vocab_file   )

						__SCREAMING_SNAKE_CASE						   =				tokenizer.tokenize("UNwant\u00E9d,running"   )
						self.assertListEqual(UpperCAmelCase__			,			["un", "##want", "##ed", ",", "runn", "##ing"]   )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__   )			,			[7, 4, 5, 1_0, 8, 9]   )




	def   UpperCAmelCase_      ( self			:				Optional[int]   )       ->	Tuple:
						__SCREAMING_SNAKE_CASE						   =				self.get_tokenizers(do_lower_case=UpperCAmelCase__   )
						for tokenizer in tokenizers:
											__SCREAMING_SNAKE_CASE						   =				tokenizer("UNwant\u00E9d,running"   )
											__SCREAMING_SNAKE_CASE						   =				len(inputs["input_ids"]   ) - 1
											self.assertListEqual(inputs["token_type_ids"]			,			[2] + [0] * sentence_len   )

											__SCREAMING_SNAKE_CASE						   =				tokenizer("UNwant\u00E9d,running"			,			"UNwant\u00E9d,running"   )
											self.assertListEqual(inputs["token_type_ids"]			,			[2] + [0] * sentence_len + [1] * sentence_len   )







"""simple docstring"""




def 						UpperCAmelCase__		(     _UpperCAmelCase					):






  """simple docstring"""





  A_  :       List[str]										=  hex_num.strip()
  if not hex_num:
    raise ValueError('No value was passed to the function'					)

  A_  :       Any										=  hex_num[0] == '-'
  if is_negative:
    A_  :       Optional[Any]										=  hex_num[1:]

  try:
    A_  :       Tuple										=  int(_UpperCAmelCase ,					16					)
  except ValueError:
    raise ValueError('Invalid value was passed to the function'					)

  A_  :       Union[str, Any]										=  ''
  while int_num > 0:
    A_  :       Optional[Any]										=  str(int_num % 2					) + bin_str
    int_num >>= 1

  return int(('-' + bin_str) if is_negative else bin_str					)


if __name__ == "__main__":
   import doctest

   doctest.testmod()