


'''simple docstring'''






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    __snake_case       (   _SCREAMING_SNAKE_CASE	,unittest.TestCase):







    """simple docstring"""



    lowercase    =					None
    lowercase    =					BloomTokenizerFast
    lowercase    =					BloomTokenizerFast
    lowercase    =					True
    lowercase    =					False
    lowercase    =					'tokenizer_file'
    lowercase    =					{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>'}

    def __lowercase   (   self    :  Any		)   ->  Optional[Any]:
      super().setUp()
      lowerCAmelCase_			:      Tuple			=   BloomTokenizerFast.from_pretrained("""bigscience/tokenizer"""		)
      tokenizer.save_pretrained(self.tmpdirname		)

    def __lowercase   (   self    :  str    ,						**lowerCamelCase    :  Dict		)   ->  List[str]:
      kwargs.update(self.special_tokens_map		)
      return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,						**lowerCamelCase		)

    def __lowercase   (   self    :  List[Any]		)   ->  List[Any]:
      lowerCAmelCase_			:      str			=   self.get_rust_tokenizer()

      lowerCAmelCase_			:      str			=   ["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
      lowerCAmelCase_			:      Dict			=   [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

      lowerCAmelCase_			:      Optional[Any]			=   tokenizer.batch_encode_plus(lowerCamelCase		)["""input_ids"""]
      self.assertListEqual(lowerCamelCase    ,						lowerCamelCase		)

      lowerCAmelCase_			:      str			=   tokenizer.batch_decode(lowerCamelCase		)
      self.assertListEqual(lowerCamelCase    ,						lowerCamelCase		)

    def __lowercase   (   self    :  Dict    ,						lowerCamelCase    :  Any=6		)   ->  List[Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
          lowerCAmelCase_			:      Optional[Any]			=   self.rust_tokenizer_class.from_pretrained(lowerCamelCase    ,						**lowerCamelCase		)
          # tokenizer_r.pad_token = None # Hotfixing padding = None
          # Simple input
          lowerCAmelCase_			:      List[str]			=   """This is a simple input"""
          lowerCAmelCase_			:      Optional[int]			=   ["""This is a simple input 1""", """This is a simple input 2"""]
          lowerCAmelCase_			:      List[Any]			=   ("""This is a simple input""", """This is a pair""")
          lowerCAmelCase_			:      Tuple			=   [
              ("""This is a simple input 1""", """This is a simple input 2"""),
              ("""This is a simple pair 1""", """This is a simple pair 2"""),
          ]

          # Simple input tests
          try:
            tokenizer_r.encode(lowerCamelCase    ,						max_length=lowerCamelCase		)
            tokenizer_r.encode_plus(lowerCamelCase    ,						max_length=lowerCamelCase		)

            tokenizer_r.batch_encode_plus(lowerCamelCase    ,						max_length=lowerCamelCase		)
            tokenizer_r.encode(lowerCamelCase    ,						max_length=lowerCamelCase		)
            tokenizer_r.batch_encode_plus(lowerCamelCase    ,						max_length=lowerCamelCase		)
          except ValueError:
            self.fail("""Bloom Tokenizer should be able to deal with padding"""		)

          lowerCAmelCase_			:      Union[str, Any]			=   None  # Hotfixing padding = None
          self.assertRaises(lowerCamelCase    ,						tokenizer_r.encode    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""		)

          # Simple input
          self.assertRaises(lowerCamelCase    ,						tokenizer_r.encode_plus    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""		)

          # Simple input
          self.assertRaises(
              lowerCamelCase    ,						tokenizer_r.batch_encode_plus    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""    ,						)

          # Pair input
          self.assertRaises(lowerCamelCase    ,						tokenizer_r.encode    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""		)

          # Pair input
          self.assertRaises(lowerCamelCase    ,						tokenizer_r.encode_plus    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""		)

          # Pair input
          self.assertRaises(
              lowerCamelCase    ,						tokenizer_r.batch_encode_plus    ,						lowerCamelCase    ,						max_length=lowerCamelCase    ,						padding="""max_length"""    ,						)

    def __lowercase   (   self    :  Any		)   ->  Tuple:
      lowerCAmelCase_			:      str			=   self.get_rust_tokenizer()
      lowerCAmelCase_			:      Dict			=   load_dataset("""xnli"""    ,						"""all_languages"""    ,						split="""test"""    ,						streaming=lowerCamelCase		)

      lowerCAmelCase_			:      Optional[int]			=   next(iter(lowerCamelCase		)		)["""premise"""]  # pick up one data
      lowerCAmelCase_			:      Optional[Any]			=   list(sample_data.values()		)

      lowerCAmelCase_			:      str			=   list(map(tokenizer.encode    ,						lowerCamelCase		)		)
      lowerCAmelCase_			:      Optional[int]			=   [tokenizer.decode(lowerCamelCase    ,						clean_up_tokenization_spaces=lowerCamelCase		) for x in output_tokens]
      self.assertListEqual(lowerCamelCase    ,						lowerCamelCase		)





    def __lowercase   (   self    :  Optional[Any]		)   ->  int:
      # The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
      # any sequence length constraints. This test of the parent class will fail since it relies on the
      # maximum sequence length of the positoonal embeddings.
      self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map		)    ,						1		)
      self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()		)[0]		)    ,						1		)







'''simple docstring'''






class    __snake_case       :







    """simple docstring"""



    def __init__(   self    :  int    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int=None    ,						lowerCamelCase    :  int=None		)   ->  str:
      lowerCAmelCase_			:      str			=   data
      lowerCAmelCase_			:      Optional[Any]			=   previous
      lowerCAmelCase_			:      int			=   next_node

    def __str__(   self    :  Any		)   ->  str:
      return F'{self.data}'

    def __lowercase   (   self    :  Optional[Any]		)   ->  int:
      return self.data

    def __lowercase   (   self    :  str		)   ->  List[str]:
      return self.next





    def __lowercase   (   self    :  int		)   ->  Optional[int]:
      return self.previous







class    __snake_case       :







    """simple docstring"""



    def __init__(   self    :  Union[str, Any]    ,						lowerCamelCase    :  Optional[int]		)   ->  Optional[Any]:
      lowerCAmelCase_			:      Optional[Any]			=   head

    def __iter__(   self    :  str		)   ->  Optional[Any]:
      return self





    def __lowercase   (   self    :  Union[str, Any]		)   ->  Dict:
      if not self.current:
        raise StopIteration
      else:
        lowerCAmelCase_			:      Dict			=   self.current.get_data()
        lowerCAmelCase_			:      Tuple			=   self.current.get_next()
        return value







class    __snake_case       :







    """simple docstring"""



    def __init__(   self    :  Union[str, Any]		)   ->  Any:
      lowerCAmelCase_			:      Optional[Any]			=   None  # First node in list
      lowerCAmelCase_			:      Optional[Any]			=   None  # Last node in list

    def __str__(   self    :  Optional[int]		)   ->  Dict:
      lowerCAmelCase_			:      str			=   self.head
      lowerCAmelCase_			:      Tuple			=   []
      while current is not None:
        nodes.append(current.get_data()		)
        lowerCAmelCase_			:      str			=   current.get_next()
      return " ".join(str(lowerCamelCase		) for node in nodes		)

    def __contains__(   self    :  List[Any]    ,						lowerCamelCase    :  int		)   ->  List[str]:
      lowerCAmelCase_			:      List[str]			=   self.head
      while current:
        if current.get_data() == value:
          return True
        lowerCAmelCase_			:      List[Any]			=   current.get_next()
      return False

    def __iter__(   self    :  str		)   ->  Optional[Any]:
      return LinkedListIterator(self.head		)

    def __lowercase   (   self    :  Dict		)   ->  Optional[int]:
      if self.head:
        return self.head.get_data()
      return None

    def __lowercase   (   self    :  List[str]		)   ->  Optional[Any]:
      if self.tail:
        return self.tail.get_data()
      return None

    def __lowercase   (   self    :  Optional[Any]    ,						lowerCamelCase    :  Node		)   ->  None:
      if self.head is None:
        lowerCAmelCase_			:      Union[str, Any]			=   node
        lowerCAmelCase_			:      List[str]			=   node
      else:
        self.insert_before_node(self.head    ,						lowerCamelCase		)

    def __lowercase   (   self    :  Tuple    ,						lowerCamelCase    :  Node		)   ->  None:
      if self.head is None:
        self.set_head(lowerCamelCase		)
      else:
        self.insert_after_node(self.tail    ,						lowerCamelCase		)

    def __lowercase   (   self    :  Union[str, Any]    ,						lowerCamelCase    :  int		)   ->  None:
      lowerCAmelCase_			:      int			=   Node(lowerCamelCase		)
      if self.head is None:
        self.set_head(lowerCamelCase		)
      else:
        self.set_tail(lowerCamelCase		)

    def __lowercase   (   self    :  Optional[Any]    ,						lowerCamelCase    :  Node    ,						lowerCamelCase    :  Node		)   ->  None:
      lowerCAmelCase_			:      Optional[int]			=   node
      lowerCAmelCase_			:      List[Any]			=   node.previous

      if node.get_previous() is None:
        lowerCAmelCase_			:      Tuple			=   node_to_insert
      else:
        lowerCAmelCase_			:      Dict			=   node_to_insert

      lowerCAmelCase_			:      Optional[int]			=   node_to_insert

    def __lowercase   (   self    :  Union[str, Any]    ,						lowerCamelCase    :  Node    ,						lowerCamelCase    :  Node		)   ->  None:
      lowerCAmelCase_			:      Optional[int]			=   node
      lowerCAmelCase_			:      Tuple			=   node.next

      if node.get_next() is None:
        lowerCAmelCase_			:      Tuple			=   node_to_insert
      else:
        lowerCAmelCase_			:      Tuple			=   node_to_insert

      lowerCAmelCase_			:      Optional[Any]			=   node_to_insert

    def __lowercase   (   self    :  Dict    ,						lowerCamelCase    :  int    ,						lowerCamelCase    :  int		)   ->  None:
      lowerCAmelCase_			:      List[str]			=   1
      lowerCAmelCase_			:      Tuple			=   Node(lowerCamelCase		)
      lowerCAmelCase_			:      List[Any]			=   self.head
      while node:
        if current_position == position:
          self.insert_before_node(lowerCamelCase    ,						lowerCamelCase		)
          return
        current_position += 1
        lowerCAmelCase_			:      str			=   node.next
      self.insert_after_node(self.tail    ,						lowerCamelCase		)

    def __lowercase   (   self    :  int    ,						lowerCamelCase    :  int		)   ->  Node:
      lowerCAmelCase_			:      List[Any]			=   self.head
      while node:
        if node.get_data() == item:
          return node
        lowerCAmelCase_			:      List[Any]			=   node.get_next()
      raise Exception("""Node not found"""		)

    def __lowercase   (   self    :  str    ,						lowerCamelCase    :  str		)   ->  int:
      if (node := self.get_node(lowerCamelCase		)) is not None:
        if node == self.head:
          lowerCAmelCase_			:      Any			=   self.head.get_next()

        if node == self.tail:
          lowerCAmelCase_			:      Optional[int]			=   self.tail.get_previous()

        self.remove_node_pointers(lowerCamelCase		)

    @staticmethod
    def __lowercase   (   lowerCamelCase    :  Node		)   ->  None:
      if node.get_next():
        lowerCAmelCase_			:      Tuple			=   node.previous

      if node.get_previous():
        lowerCAmelCase_			:      Any			=   node.next

      lowerCAmelCase_			:      List[Any]			=   None
      lowerCAmelCase_			:      Any			=   None





    def __lowercase   (   self    :  str		)   ->  Optional[Any]:
      return self.head is None







def 				UpperCamelCase_			(    ):
  '''simple docstring'''







if __name__ == "__main__":
  import doctest

  doctest.testmod()



