






import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command


class    __lowercase   (						unittest.TestCase	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					inspect.getfile(accelerate.test_utils	)
     SCREAMING_SNAKE_CASE     =					os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ["scripts", "test_cli.py"]	)

     SCREAMING_SNAKE_CASE     =					["accelerate", "launch"]
     SCREAMING_SNAKE_CASE     =					Path.home() / ".cache/huggingface/accelerate"
     SCREAMING_SNAKE_CASE     =					"default_config.yaml"
     SCREAMING_SNAKE_CASE     =					config_folder / config_file
     SCREAMING_SNAKE_CASE     =					config_folder / "_default_config.yaml"

     SCREAMING_SNAKE_CASE     =					Path("tests/test_configs"	)




     @classmethod
     def 		lowerCAmelCase_  (					cls							:      int					):


         """simple docstring"""



         if cls.config_path.is_file():
             cls.config_path.rename(cls.changed_path					)




     @classmethod
     def 		lowerCAmelCase_  (					cls							:      Tuple					):


         """simple docstring"""



         if cls.changed_path.is_file():
             cls.changed_path.rename(cls.config_path					)




     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						self.base_cmd
         if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
             cmd += ["--multi_gpu"]
         execute_subprocess_async(cmd + [self.test_file_path]			,						env=os.environ.copy()					)




     def 		lowerCAmelCase_  (					self							:      Dict					):


         """simple docstring"""



         for config in sorted(self.test_config_path.glob("""**/*.yaml"""					)					):
             with self.subTest(config_file=UpperCamelCase_					):
                 execute_subprocess_async(
                     self.base_cmd + ["""--config_file""", str(UpperCamelCase_					), self.test_file_path]			,						env=os.environ.copy()					)





     def 		lowerCAmelCase_  (					self							:      Dict					):


         """simple docstring"""



         execute_subprocess_async(["""accelerate""", """test"""]			,						env=os.environ.copy()					)


class    __lowercase   (						unittest.TestCase	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					"test-tpu"
     SCREAMING_SNAKE_CASE     =					"us-central1-a"
     SCREAMING_SNAKE_CASE     =					"ls"
     SCREAMING_SNAKE_CASE     =					["accelerate", "tpu-config"]
     SCREAMING_SNAKE_CASE     =					"cd /usr/share"
     SCREAMING_SNAKE_CASE     =					"tests/test_samples/test_command_file.sh"
     SCREAMING_SNAKE_CASE     =					"Running gcloud compute tpus tpu-vm ssh"




     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + ["""--command""", self.command, """--tpu_zone""", self.tpu_zone, """--tpu_name""", self.tpu_name, """--debug"""]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      List[Any]					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + [
                 """--config_file""",
                 """tests/test_configs/0_12_0.yaml""",
                 """--command""",
                 self.command,
                 """--tpu_zone""",
                 self.tpu_zone,
                 """--tpu_name""",
                 self.tpu_name,
                 """--debug""",
             ]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      Any					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd + ["""--config_file""", """tests/test_configs/latest.yaml""", """--debug"""]			,						return_stdout=UpperCamelCase_					)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      int					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd + ["""--config_file""", """tests/test_configs/latest.yaml""", """--command""", self.command, """--debug"""]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      Optional[Any]					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + [
                 """--config_file""",
                 """tests/test_configs/latest.yaml""",
                 """--command""",
                 self.command,
                 """--command""",
                 """echo \"Hello World\"""",
                 """--debug""",
             ]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo \"Hello World\" --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + ["""--config_file""", """tests/test_configs/latest.yaml""", """--command_file""", self.command_file, """--debug"""]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      int					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + [
                 """--config_file""",
                 """tests/test_configs/0_12_0.yaml""",
                 """--command_file""",
                 self.command_file,
                 """--tpu_zone""",
                 self.tpu_zone,
                 """--tpu_name""",
                 self.tpu_name,
                 """--debug""",
             ]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all"			,						UpperCamelCase_			,						)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd + ["""--config_file""", """tests/test_configs/latest.yaml""", """--install_accelerate""", """--debug"""]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo \"hello world\"; echo \"this is a second command\" --worker all"			,						UpperCamelCase_			,						)





     def 		lowerCAmelCase_  (					self							:      Optional[int]					):


         """simple docstring"""



         __A       =						run_command(
             self.cmd
             + [
                 """--config_file""",
                 """tests/test_configs/latest.yaml""",
                 """--install_accelerate""",
                 """--accelerate_version""",
                 """12.0.0""",
                 """--debug""",
             ]			,						return_stdout=UpperCamelCase_			,						)
         self.assertIn(
             F"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo \"hello world\"; echo \"this is a second command\" --worker all"			,						UpperCamelCase_			,						)












import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__a   :							Union[str, Any]						= logging.get_logger(__name__)

__a   :							List[str]						= {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__a   :							Dict						= {
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

__a   :							List[str]						= {
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE     =					PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE     =					["input_ids", "attention_mask"]
     SCREAMING_SNAKE_CASE     =					RobertaTokenizer




     def __init__(					self							:      List[Any]			,						UpperCamelCase_							:      Any=None			,						UpperCamelCase_							:      Tuple=None			,						UpperCamelCase_							:      Optional[int]=None			,						UpperCamelCase_							:      Union[str, Any]="replace"			,						UpperCamelCase_							:      Any="<s>"			,						UpperCamelCase_							:      Tuple="</s>"			,						UpperCamelCase_							:      Optional[Any]="</s>"			,						UpperCamelCase_							:      Dict="<s>"			,						UpperCamelCase_							:      int="<unk>"			,						UpperCamelCase_							:      Optional[int]="<pad>"			,						UpperCamelCase_							:      str="<mask>"			,						UpperCamelCase_							:      Tuple=False			,						UpperCamelCase_							:      Optional[Any]=True			,						**UpperCamelCase_							:      Union[str, Any]			,						):


         """simple docstring"""



         super().__init__(
             UpperCamelCase_			,						UpperCamelCase_			,						tokenizer_file=UpperCamelCase_			,						errors=UpperCamelCase_			,						bos_token=UpperCamelCase_			,						eos_token=UpperCamelCase_			,						sep_token=UpperCamelCase_			,						cls_token=UpperCamelCase_			,						unk_token=UpperCamelCase_			,						pad_token=UpperCamelCase_			,						mask_token=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_			,						trim_offsets=UpperCamelCase_			,						**UpperCamelCase_			,						)

         __A       =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
         if pre_tok_state.get("""add_prefix_space"""			,						UpperCamelCase_					) != add_prefix_space:
             __A       =						getattr(UpperCamelCase_			,						pre_tok_state.pop("""type"""					)					)
             __A       =						add_prefix_space
             __A       =						pre_tok_class(**UpperCamelCase_					)

         __A       =						add_prefix_space

         __A       =						"""post_processor"""
         __A       =						getattr(self.backend_tokenizer			,						UpperCamelCase_			,						UpperCamelCase_					)
         if tokenizer_component_instance:
             __A       =						json.loads(tokenizer_component_instance.__getstate__()					)

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                 __A       =						tuple(state["""sep"""]					)
             if "cls" in state:
                 __A       =						tuple(state["""cls"""]					)

             __A       =						False

             if state.get("""add_prefix_space"""			,						UpperCamelCase_					) != add_prefix_space:
                 __A       =						add_prefix_space
                 __A       =						True

             if state.get("""trim_offsets"""			,						UpperCamelCase_					) != trim_offsets:
                 __A       =						trim_offsets
                 __A       =						True

             if changes_to_apply:
                 __A       =						getattr(UpperCamelCase_			,						state.pop("""type"""					)					)
                 __A       =						component_class(**UpperCamelCase_					)
                 setattr(self.backend_tokenizer			,						UpperCamelCase_			,						UpperCamelCase_					)




     @property
     def 		lowerCAmelCase_  (					self							:      str					):


         """simple docstring"""



         if self._mask_token is None:
             if self.verbose:
                 logger.error("""Using mask_token, but it is not set yet."""					)
             return None
         return str(self._mask_token					)




     @mask_token.setter
     def 		lowerCAmelCase_  (					self							:      str			,						UpperCamelCase_							:      Tuple					):


         """simple docstring"""



         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else value
         __A       =						value




     def 		lowerCAmelCase_  (					self							:      Tuple			,						*UpperCamelCase_							:      Optional[Any]			,						**UpperCamelCase_							:      Dict					):


         """simple docstring"""



         __A       =						kwargs.get("""is_split_into_words"""			,						UpperCamelCase_					)
         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*UpperCamelCase_			,						**UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]			,						*UpperCamelCase_							:      Optional[int]			,						**UpperCamelCase_							:      Tuple					):


         """simple docstring"""



         __A       =						kwargs.get("""is_split_into_words"""			,						UpperCamelCase_					)

         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*UpperCamelCase_			,						**UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      int			,						UpperCamelCase_							:      str			,						UpperCamelCase_							:      Optional[str] = None					):


         """simple docstring"""



         __A       =						self._tokenizer.model.save(UpperCamelCase_			,						name=UpperCamelCase_					)
         return tuple(UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]			,						UpperCamelCase_							:      Any			,						UpperCamelCase_							:      int=None					):


         """simple docstring"""



         __A       =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
             return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





     def 		lowerCAmelCase_  (					self							:      int			,						UpperCamelCase_							:      List[int]			,						UpperCamelCase_							:      Optional[List[int]] = None					):


         """simple docstring"""



         __A       =						[self.sep_token_id]
         __A       =						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep					) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]




