


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase							:   Optional[Any]						=						get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")






@require_sentencepiece
@require_tokenizers
class    _UpperCamelCase     (       __snake_case     ,		unittest.TestCase					):

				"""simple docstring"""







				lowerCAmelCase					=			GPTSwaTokenizer
				lowerCAmelCase					=			False
				lowerCAmelCase					=			True
				lowerCAmelCase					=			False

				def   _UpperCAmelCase       (    self )						->					Dict:
					super().setUp()

					# We have a SentencePiece fixture for testing
					A             =					GPTSwaTokenizer(a__					, eos_token="""<unk>"""					, bos_token="""<unk>"""					, pad_token="""<unk>""" )

					tokenizer.save_pretrained(self.tmpdirname )

				def   _UpperCAmelCase       (    self					, a__ )						->					Tuple:
					A             =					"""This is a test"""
					A             =					"""This is a test"""
					return input_text, output_text

				def   _UpperCAmelCase       (    self )						->					List[str]:
					A             =					"""<s>"""
					A             =					1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__ )					, a__ )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__ )					, a__ )

				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					list(self.get_tokenizer().get_vocab().keys() )

					self.assertEqual(vocab_keys[0]					, """<unk>""" )
					self.assertEqual(vocab_keys[1]					, """<s>""" )
					self.assertEqual(vocab_keys[-1]					, """j""" )
					self.assertEqual(len(a__ )					, 2000 )

				def   _UpperCAmelCase       (    self )						->					Optional[int]:
					self.assertEqual(self.get_tokenizer().vocab_size					, 2000 )

				def   _UpperCAmelCase       (    self )						->					Dict:
					A             =					GPTSwaTokenizer(a__ )

					A             =					tokenizer.tokenize("""This is a test""" )
					self.assertListEqual(a__					, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

					self.assertListEqual(tokenizer.convert_tokens_to_ids(a__ )					, [465, 287, 265, 631, 842] )

					A             =					tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
					# fmt: off
					self.assertListEqual(
					    a__					, ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]					, )
					# fmt: on

					A             =					tokenizer.convert_tokens_to_ids(a__ )
					self.assertListEqual(
					    a__					, [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					, )

					A             =					tokenizer.convert_ids_to_tokens(a__ )
					# fmt: off
					self.assertListEqual(
					    a__					, ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""] )
					# fmt: on

				def   _UpperCAmelCase       (    self )						->					Dict:
					A             =					GPTSwaTokenizer(a__ )
					A             =					["""This is a test""", """I was born in 92000, and this is falsé."""]
					A             =					[
					    [465, 287, 265, 631, 842],
					    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
					]

					# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
					for text, expected_ids in zip(a__					, a__ ):
						self.assertListEqual(tokenizer.encode_fast(a__ )					, a__ )

					# Test that decode_fast returns the input text
					for text, token_ids in zip(a__					, a__ ):
						self.assertEqual(tokenizer.decode_fast(a__ )					, a__ )

				@slow
				def   _UpperCAmelCase       (    self )						->					Any:
					A             =					[
					    """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
					    """Hey there, how are you doing this fine day?""",
					    """This is a text with a trailing spaces followed by a dot     .""",
					    """Häj sväjs lillebrör! =)""",
					    """Det är inget fel på Mr. Cool""",
					]

					# fmt: off
					A             =					{"""input_ids""": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
					# fmt: on
					self.tokenizer_integration_test_util(
					    expected_encoding=a__					, model_name="""AI-Sweden/gpt-sw3-126m"""					, sequences=a__					, )







_lowercase							:   Dict						=						"0.21.0"

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
				from .utils import rich



