




'''simple docstring'''




import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_      :		Any							      =       get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")


@require_sentencepiece
@require_tokenizers
class   __UpperCamelCase       ( _A			,					unittest.TestCase				):
       lowercase	:    Tuple      						=GPTSwaTokenizer
       lowercase	:    str      						=False
       lowercase	:    Any      						=True
       lowercase	:    List[str]      						=False






       def      lowercase__     (							self  ):



              """simple docstring"""







              super().setUp()

              # We have a SentencePiece fixture for testing
              lowerCamelCase_							=GPTSwaTokenizer(UpperCamelCase__,		eos_token='''<unk>''',		bos_token='''<unk>''',		pad_token='''<unk>'''  )

              tokenizer.save_pretrained(self.tmpdirname  )






       def      lowercase__     (							self,		lowerCAmelCase  ):



              """simple docstring"""







              lowerCamelCase_							='''This is a test'''
              lowerCamelCase_							='''This is a test'''
              return input_text, output_text






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							='''<s>'''
              lowerCamelCase_							=1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__  ),		UpperCamelCase__  )
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__  ),		UpperCamelCase__  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=list(self.get_tokenizer().get_vocab().keys()  )

              self.assertEqual(vocab_keys[0],		'''<unk>'''  )
              self.assertEqual(vocab_keys[1],		'''<s>'''  )
              self.assertEqual(vocab_keys[-1],		'''j'''  )
              self.assertEqual(len(UpperCamelCase__  ),		2_000  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              self.assertEqual(self.get_tokenizer().vocab_size,		2_000  )






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=GPTSwaTokenizer(UpperCamelCase__  )

              lowerCamelCase_							=tokenizer.tokenize('''This is a test'''  )
              self.assertListEqual(UpperCamelCase__,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

              self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  ),		[465, 287, 265, 631, 842]  )

              lowerCamelCase_							=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
              # fmt: off
              self.assertListEqual(
                  UpperCamelCase__,		['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''],		)
              # fmt: on

              lowerCamelCase_							=tokenizer.convert_tokens_to_ids(UpperCamelCase__  )
              self.assertListEqual(
                  UpperCamelCase__,		[262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],		)

              lowerCamelCase_							=tokenizer.convert_ids_to_tokens(UpperCamelCase__  )
              # fmt: off
              self.assertListEqual(
                  UpperCamelCase__,		['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']  )
              # fmt: on






       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=GPTSwaTokenizer(UpperCamelCase__  )
              lowerCamelCase_							=['''This is a test''', '''I was born in 92000, and this is falsé.''']
              lowerCamelCase_							=[
                  [465, 287, 265, 631, 842],
                  [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
              ]

              # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
              for text, expected_ids in zip(UpperCamelCase__,		UpperCamelCase__  ):
                     self.assertListEqual(tokenizer.encode_fast(UpperCamelCase__  ),		UpperCamelCase__  )

              # Test that decode_fast returns the input text
              for text, token_ids in zip(UpperCamelCase__,		UpperCamelCase__  ):
                     self.assertEqual(tokenizer.decode_fast(UpperCamelCase__  ),		UpperCamelCase__  )






       @slow
       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=[
                  '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
                  '''Hey there, how are you doing this fine day?''',
                  '''This is a text with a trailing spaces followed by a dot     .''',
                  '''Häj sväjs lillebrör! =)''',
                  '''Det är inget fel på Mr. Cool''',
              ]

              # fmt: off
              lowerCamelCase_							={'''input_ids''': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
              # fmt: on
              self.tokenizer_integration_test_util(
                  expected_encoding=UpperCamelCase__,		model_name='''AI-Sweden/gpt-sw3-126m''',		sequences=UpperCamelCase__,		)






from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCAmelCase			:     Any           =   {
    'configuration_mctct': ['MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MCTCTConfig'],
    'feature_extraction_mctct': ['MCTCTFeatureExtractor'],
    'processing_mctct': ['MCTCTProcessor'],
}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCAmelCase			:     int           =   [
        'MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST',
        'MCTCTForCTC',
        'MCTCTModel',
        'MCTCTPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
    from .feature_extraction_mctct import MCTCTFeatureExtractor
    from .processing_mctct import MCTCTProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
    import sys

    __lowerCAmelCase			:     List[Any]           =   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

