"""simple docstring"""


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__     						=  get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")


@require_sentencepiece
@require_tokenizers
class 				a (						lowerCAmelCase_		,							unittest.TestCase      ):
    _snake_case   :     List[Any]   							=							GPTSwaTokenizer
    _snake_case   :     List[Any]   							=							False
    _snake_case   :     Union[str, Any]   							=							True
    _snake_case   :     Union[str, Any]   							=							False


    def    lowerCAmelCase_     (    self		:     str    ):
      super().setUp()

      # We have a SentencePiece fixture for testing
      _UpperCAmelCase         =					GPTSwaTokenizer(__lowerCAmelCase      , eos_token="""<unk>"""      , bos_token="""<unk>"""      , pad_token="""<unk>"""    )

      tokenizer.save_pretrained(self.tmpdirname    )


    def    lowerCAmelCase_     (    self		:     Optional[int]      , __lowerCAmelCase		:     Union[str, Any]    ):
      _UpperCAmelCase         =					"""This is a test"""
      _UpperCAmelCase         =					"""This is a test"""
      return input_text, output_text


    def    lowerCAmelCase_     (    self		:     Optional[int]    ):
      _UpperCAmelCase         =					"""<s>"""
      _UpperCAmelCase         =					1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase    )      , __lowerCAmelCase    )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase    )      , __lowerCAmelCase    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase         =					list(self.get_tokenizer().get_vocab().keys()    )

      self.assertEqual(vocab_keys[0]      , """<unk>"""    )
      self.assertEqual(vocab_keys[1]      , """<s>"""    )
      self.assertEqual(vocab_keys[-1]      , """j"""    )
      self.assertEqual(len(__lowerCAmelCase    )      , 2000    )


    def    lowerCAmelCase_     (    self		:     Any    ):
      self.assertEqual(self.get_tokenizer().vocab_size      , 2000    )


    def    lowerCAmelCase_     (    self		:     str    ):
      _UpperCAmelCase         =					GPTSwaTokenizer(__lowerCAmelCase    )

      _UpperCAmelCase         =					tokenizer.tokenize("""This is a test"""    )
      self.assertListEqual(__lowerCAmelCase      , ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase    )      , [465, 287, 265, 631, 842]    )

      _UpperCAmelCase         =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
      # fmt: off
      self.assertListEqual(
          __lowerCAmelCase      , ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]      , )
      # fmt: on

      _UpperCAmelCase         =					tokenizer.convert_tokens_to_ids(__lowerCAmelCase    )
      self.assertListEqual(
          __lowerCAmelCase      , [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]      , )

      _UpperCAmelCase         =					tokenizer.convert_ids_to_tokens(__lowerCAmelCase    )
      # fmt: off
      self.assertListEqual(
          __lowerCAmelCase      , ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]    )
      # fmt: on


    def    lowerCAmelCase_     (    self		:     Any    ):
      _UpperCAmelCase         =					GPTSwaTokenizer(__lowerCAmelCase    )
      _UpperCAmelCase         =					["""This is a test""", """I was born in 92000, and this is falsé."""]
      _UpperCAmelCase         =					[
          [465, 287, 265, 631, 842],
          [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
      ]

      # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
      for text, expected_ids in zip(__lowerCAmelCase      , __lowerCAmelCase    ):
        self.assertListEqual(tokenizer.encode_fast(__lowerCAmelCase    )      , __lowerCAmelCase    )

      # Test that decode_fast returns the input text
      for text, token_ids in zip(__lowerCAmelCase      , __lowerCAmelCase    ):
        self.assertEqual(tokenizer.decode_fast(__lowerCAmelCase    )      , __lowerCAmelCase    )


    @slow
    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase         =					[
          """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
          """Hey there, how are you doing this fine day?""",
          """This is a text with a trailing spaces followed by a dot     .""",
          """Häj sväjs lillebrör! =)""",
          """Det är inget fel på Mr. Cool""",
      ]

      # fmt: off
      _UpperCAmelCase         =					{"""input_ids""": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
      # fmt: on
      self.tokenizer_integration_test_util(
          expected_encoding=__lowerCAmelCase      , model_name="""AI-Sweden/gpt-sw3-126m"""      , sequences=__lowerCAmelCase      , )

"""simple docstring"""


from __future__ import annotations






def 			__UpperCAmelCase    (	lowercase      ,lowercase						):







  """simple docstring"""
  _UpperCAmelCase         =					get_failure_array(lowercase						)

  # 2) Step through text searching for pattern
  _UpperCAmelCase       , _UpperCAmelCase         =					0, 0  # index into text, pattern
  while i < len(lowercase						):
    if pattern[j] == text[i]:
      if j == (len(lowercase						) - 1):
        return True
      j += 1

    # if this is a prefix in our pattern
    # just go back far enough to continue
    elif j > 0:
      _UpperCAmelCase         =					failure[j - 1]
      continue
    i += 1
  return False






def 			__UpperCAmelCase    (	lowercase						):







  """simple docstring"""
  _UpperCAmelCase         =					[0]
  _UpperCAmelCase         =					0
  _UpperCAmelCase         =					1
  while j < len(lowercase						):
    if pattern[i] == pattern[j]:
      i += 1
    elif i > 0:
      _UpperCAmelCase         =					failure[i - 1]
      continue
    j += 1
    failure.append(lowercase						)
  return failure


if __name__ == "__main__":
    # Test 1)
    UpperCAmelCase__     						=  """abc1abc12"""
    UpperCAmelCase__     						=  """alskfjaldsabc1abc1abc12k23adsfabcabc"""
    UpperCAmelCase__     						=  """alskfjaldsk23adsfabcabc"""
    assert kmp(pattern, texta) and not kmp(pattern, texta)

    # Test 2)
    UpperCAmelCase__     						=  """ABABX"""
    UpperCAmelCase__     						=  """ABABZABABYABABX"""
    assert kmp(pattern, text)

    # Test 3)
    UpperCAmelCase__     						=  """AAAB"""
    UpperCAmelCase__     						=  """ABAAAAAB"""
    assert kmp(pattern, text)

    # Test 4)
    UpperCAmelCase__     						=  """abcdabcy"""
    UpperCAmelCase__     						=  """abcxabcdabxabcdabcdabcy"""
    assert kmp(pattern, text)

    # Test 5)
    UpperCAmelCase__     						=  """aabaabaaa"""
    assert get_failure_array(pattern) == [0, 1, 0, 1, 2, 3, 4, 5, 2]
