
'''simple docstring'''







import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 						__UpperCAmelCase      (					_lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       GPTSanJapaneseTokenizer
							__lowercase      		=       False
							__lowercase      		=       {"""do_clean_text""": False, """add_prefix_space""": False}


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().setUp()

								# fmt: off
								_snake_case   						=		['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
								# fmt: on
								_snake_case   						=		{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
								_snake_case   						=		{'unk_token': '<unk>'}

								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['vocab_file']  )
								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['emoji_file']  )
								with open(self.vocab_file    ,   'w'    ,   encoding='utf-8'  ) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
								with open(self.emoji_file    ,   'w'  ) as emoji_writer:
									emoji_writer.write(json.dumps(lowerCAmelCase_  )  )


							def  lowerCamelCase				(       self    ,   **lowerCAmelCase_  ):





								"""simple docstring"""






								kwargs.update(self.special_tokens_map  )
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname    ,   **lowerCAmelCase_  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		'こんにちは、世界。 \nこんばんは、㔺界。😀'
								_snake_case   						=		'こんにちは、世界。 \nこんばんは、世界。😀'
								return input_text, output_text


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case						, _snake_case   						=		self.get_input_output_texts(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_    ,   add_special_tokens=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_    ,   clean_up_tokenization_spaces=lowerCAmelCase_  )
								return text, ids


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass  # TODO add if relevant


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.get_tokenizer()

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。　こんばんは、㔺界。'
								_snake_case   						=		['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
								_snake_case   						=		tokenizer.tokenize(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )

								# Testing conversion to ids without special tokens
								_snake_case   						=		[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								_snake_case   						=		tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )

								# Testing conversion to ids with special tokens
								_snake_case   						=		tokens + [tokenizer.unk_token]
								_snake_case   						=		[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								_snake_case   						=		tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.get_tokenizer()

								# Testing tokenization
								_snake_case   						=		'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
								_snake_case   						=		'こんにちは、、、、世界。こんばんは、、、、世界。'
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。'
								_snake_case   						=		'こんばんは、㔺界。😀'
								_snake_case   						=		'こんにちは、世界。こんばんは、世界。😀'
								_snake_case   						=		tokenizer.encode(prefix_text + input_text  )
								_snake_case   						=		tokenizer.encode(''    ,   prefix_text=prefix_text + input_text  )
								_snake_case   						=		tokenizer.encode(lowerCAmelCase_    ,   prefix_text=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								_snake_case   						=		tokenizer.decode(lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								_snake_case   						=		'こんにちは、世界。'
								_snake_case   						=		'こんばんは、㔺界。😀'

								_snake_case   						=		len(tokenizer.encode(lowerCAmelCase_  )  ) - 2
								_snake_case   						=		len(tokenizer.encode(lowerCAmelCase_  )  ) - 2

								_snake_case   						=		[1] + [0] * (len_prefix + len_text + 1)
								_snake_case   						=		[1] * (len_prefix + len_text + 1) + [0]
								_snake_case   						=		[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								_snake_case   						=		tokenizer(prefix_text + input_text  ).token_type_ids
								_snake_case   						=		tokenizer(''    ,   prefix_text=prefix_text + input_text  ).token_type_ids
								_snake_case   						=		tokenizer(lowerCAmelCase_    ,   prefix_text=lowerCAmelCase_  ).token_type_ids
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertListEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								_snake_case   						=		tokenizer.encode('あンいワ'  )
								_snake_case   						=		tokenizer.encode(''    ,   prefix_text='あンいワ'  )
								_snake_case   						=		tokenizer.encode('いワ'    ,   prefix_text='あン'  )

								self.assertEqual(tokenizer.decode(lowerCAmelCase_  )    ,   tokenizer.decode(lowerCAmelCase_  )  )
								self.assertEqual(tokenizer.decode(lowerCAmelCase_  )    ,   tokenizer.decode(lowerCAmelCase_  )  )
								self.assertNotEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertNotEqual(lowerCAmelCase_    ,   lowerCAmelCase_  )
								self.assertEqual(x_token_a[1]    ,   x_token_a[-1]  )  # SEG token
								self.assertEqual(x_token_a[1]    ,   x_token_a[3]  )  # SEG token


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								_snake_case   						=		[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
								_snake_case   						=		tokenizer(lowerCAmelCase_    ,   padding=lowerCAmelCase_  )
								_snake_case   						=		tokenizer.batch_encode_plus(lowerCAmelCase_    ,   padding=lowerCAmelCase_  )

								# fmt: off
								_snake_case   						=		[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
								_snake_case   						=		[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								_snake_case   						=		[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token.token_type_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token.attention_mask    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.input_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.token_type_ids    ,   lowerCAmelCase_  )
								self.assertListEqual(x_token_a.attention_mask    ,   lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass




							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								pass



'''simple docstring'''







import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command







class UpperCAmelCase__	( unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				Optional[Any]   =					inspect.getfile(accelerate.test_utils	)
  __UpperCAmelCase				:				List[str]   =					os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ['''scripts''', '''test_cli.py''']	)

  __UpperCAmelCase				:				Dict   =					['''accelerate''', '''launch''']
  __UpperCAmelCase				:				Dict   =					Path.home() / '''.cache/huggingface/accelerate'''
  __UpperCAmelCase				:				Dict   =					'''default_config.yaml'''
  __UpperCAmelCase				:				Optional[Any]   =					config_folder / config_file
  __UpperCAmelCase				:				Dict   =					config_folder / '''_default_config.yaml'''

  __UpperCAmelCase				:				Any   =					Path('''tests/test_configs'''	)


  @classmethod
  def   __lowercase		(		cls      :     int       ):


     '''simple docstring'''






     if cls.config_path.is_file():
        cls.config_path.rename(cls.changed_path       )


  @classmethod
  def   __lowercase		(		cls      :     List[Any]       ):


     '''simple docstring'''






     if cls.changed_path.is_file():
        cls.changed_path.rename(cls.config_path       )


  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     _a				:	Dict			=   self.base_cmd
     if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
        cmd += ["--multi_gpu"]
     execute_subprocess_async(cmd + [self.test_file_path]							,env=os.environ.copy()       )


  def   __lowercase		(		self      :     Optional[Any]       ):


     '''simple docstring'''






     for config in sorted(self.test_config_path.glob('**/*.yaml'       )       ):
        with self.subTest(config_file=_a       ):
           execute_subprocess_async(
               self.base_cmd + ['--config_file', str(_a       ), self.test_file_path]							,env=os.environ.copy()       )


  def   __lowercase		(		self      :     Optional[int]       ):


     '''simple docstring'''






     execute_subprocess_async(['accelerate', 'test']							,env=os.environ.copy()       )







class UpperCAmelCase__	( unittest.TestCase	):





  """simple docstring"""

  __UpperCAmelCase				:				Optional[Any]   =					'''test-tpu'''
  __UpperCAmelCase				:				Any   =					'''us-central1-a'''
  __UpperCAmelCase				:				List[Any]   =					'''ls'''
  __UpperCAmelCase				:				Any   =					['''accelerate''', '''tpu-config''']
  __UpperCAmelCase				:				Dict   =					'''cd /usr/share'''
  __UpperCAmelCase				:				Any   =					'''tests/test_samples/test_command_file.sh'''
  __UpperCAmelCase				:				List[Any]   =					'''Running gcloud compute tpus tpu-vm ssh'''


  def   __lowercase		(		self      :     Dict       ):


     '''simple docstring'''






     _a				:	Optional[Any]			=   run_command(
         self.cmd
         + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Any			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/0_12_0.yaml',
             '--command',
             self.command,
             '--tpu_zone',
             self.tpu_zone,
             '--tpu_name',
             self.tpu_name,
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug']							,return_stdout=_a       )
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     int       ):


     '''simple docstring'''






     _a				:	Optional[Any]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"""							,_a							,)


  def   __lowercase		(		self      :     str       ):


     '''simple docstring'''






     _a				:	List[str]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/latest.yaml',
             '--command',
             self.command,
             '--command',
             'echo "Hello World"',
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo \"Hello World\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Union[str, Any]       ):


     '''simple docstring'''






     _a				:	Any			=   run_command(
         self.cmd
         + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Union[str, Any]       ):


     '''simple docstring'''






     _a				:	Union[str, Any]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/0_12_0.yaml',
             '--command_file',
             self.command_file,
             '--tpu_zone',
             self.tpu_zone,
             '--tpu_name',
             self.tpu_name,
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     Any       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug']							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)


  def   __lowercase		(		self      :     List[str]       ):


     '''simple docstring'''






     _a				:	Optional[int]			=   run_command(
         self.cmd
         + [
             '--config_file',
             'tests/test_configs/latest.yaml',
             '--install_accelerate',
             '--accelerate_version',
             '12.0.0',
             '--debug',
         ]							,return_stdout=_a							,)
     self.assertIn(
         F"""{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo \"hello world\"; echo \"this is a second command\" --worker all"""							,_a							,)



