






"""simple docstring"""



import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast





class 					_UpperCAmelCase       ( unittest.TestCase							):




					def   A  (				self     :						Union[str, Any]   )  ->  Tuple:
							# A mock response for an HTTP head request to emulate server down
							lowercase_ : List[str]							       =   mock.Mock()
							lowercase_ : Optional[int]							       =   5_00
							lowercase_ : str							       =   {}
							lowercase_ : Dict							       =   HTTPError
							lowercase_ : Tuple							       =   {}

							# Download this model to make sure it's in the cache.
							lowercase_ : Optional[Any]							       =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('''requests.Session.request'''    ,						return_value=A   ) as mock_head:
									lowercase_ : int							       =   BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''   )
									# This check we did call the fake head request
									mock_head.assert_called()




					@require_tokenizers
					def   A  (				self     :						str   )  ->  int:
							# A mock response for an HTTP head request to emulate server down
							lowercase_ : List[str]							       =   mock.Mock()
							lowercase_ : Any							       =   5_00
							lowercase_ : Union[str, Any]							       =   {}
							lowercase_ : int							       =   HTTPError
							lowercase_ : Any							       =   {}

							# Download this model to make sure it's in the cache.
							lowercase_ : Union[str, Any]							       =   GPTaTokenizerFast.from_pretrained('''gpt2'''   )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch('''requests.Session.request'''    ,						return_value=A   ) as mock_head:
									lowercase_ : int							       =   GPTaTokenizerFast.from_pretrained('''gpt2'''   )
									# This check we did call the fake head request
									mock_head.assert_called()




					def   A  (				self     :						List[Any]   )  ->  Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							try:
									lowercase_ : List[str]							       =   tempfile.mktemp()
									with open(A    ,						'''wb'''   ) as f:
											http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''    ,						A   )

									lowercase_ : int							       =   AlbertTokenizer.from_pretrained(A   )
							finally:
									os.remove(A   )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile('''tokenizer.json'''   ):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
							try:
									with open('''tokenizer.json'''    ,						'''wb'''   ) as f:
											http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''    ,						A   )
									lowercase_ : int							       =   AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''   )
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size    ,						10_00   )
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
									os.remove('''tokenizer.json'''   )






					def   A  (				self     :						Optional[Any]   )  ->  Optional[int]:
							# This test is for deprecated behavior and can be removed in v5
							lowercase_ : Tuple							       =   AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''   )





@is_staging_test
class 					_UpperCAmelCase       ( unittest.TestCase							):
					SCREAMING_SNAKE_CASE_		:				Optional[int]							    =   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]




					@classmethod
					def   A  (				cls     :						Optional[Any]   )  ->  Optional[int]:
							lowercase_ : str							       =   TOKEN
							HfFolder.save_token(A   )




					@classmethod
					def   A  (				cls     :						Union[str, Any]   )  ->  Union[str, Any]:
							try:
									delete_repo(token=cls._token    ,						repo_id='''test-tokenizer'''   )
							except HTTPError:
									pass

							try:
									delete_repo(token=cls._token    ,						repo_id='''valid_org/test-tokenizer-org'''   )
							except HTTPError:
									pass

							try:
									delete_repo(token=cls._token    ,						repo_id='''test-dynamic-tokenizer'''   )
							except HTTPError:
									pass




					def   A  (				self     :						Optional[int]   )  ->  Optional[Any]:
							with tempfile.TemporaryDirectory() as tmp_dir:
									lowercase_ : Tuple							       =   os.path.join(A    ,						'''vocab.txt'''   )
									with open(A    ,						'''w'''    ,						encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
									lowercase_ : Dict							       =   BertTokenizer(A   )

							tokenizer.push_to_hub('''test-tokenizer'''    ,						use_auth_token=self._token   )
							lowercase_ : int							       =   BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''   )
							self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab   )

							# Reset repo
							delete_repo(token=self._token    ,						repo_id='''test-tokenizer'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(A    ,						repo_id='''test-tokenizer'''    ,						push_to_hub=A    ,						use_auth_token=self._token   )

							lowercase_ : int							       =   BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''   )
							self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab   )




					def   A  (				self     :						Optional[int]   )  ->  int:
							with tempfile.TemporaryDirectory() as tmp_dir:
									lowercase_ : Dict							       =   os.path.join(A    ,						'''vocab.txt'''   )
									with open(A    ,						'''w'''    ,						encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
									lowercase_ : Union[str, Any]							       =   BertTokenizer(A   )

							tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''    ,						use_auth_token=self._token   )
							lowercase_ : Union[str, Any]							       =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
							self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab   )

							# Reset repo
							delete_repo(token=self._token    ,						repo_id='''valid_org/test-tokenizer-org'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    A    ,						repo_id='''valid_org/test-tokenizer-org'''    ,						push_to_hub=A    ,						use_auth_token=self._token   )

							lowercase_ : Dict							       =   BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''   )
							self.assertDictEqual(new_tokenizer.vocab    ,						tokenizer.vocab   )






					@require_tokenizers
					def   A  (				self     :						Optional[Any]   )  ->  List[Any]:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
									lowercase_ : List[str]							       =   os.path.join(A    ,						'''vocab.txt'''   )
									with open(A    ,						'''w'''    ,						encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )
									lowercase_ : Any							       =   CustomTokenizer(A   )

							# No fast custom tokenizer
							tokenizer.push_to_hub('''test-dynamic-tokenizer'''    ,						use_auth_token=self._token   )

							lowercase_ : Union[str, Any]							       =   AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''    ,						trust_remote_code=A   )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizer'''   )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
									lowercase_ : Optional[Any]							       =   os.path.join(A    ,						'''vocab.txt'''   )
									with open(A    ,						'''w'''    ,						encoding='''utf-8'''   ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]   )   )

									lowercase_ : List[Any]							       =   BertTokenizerFast.from_pretrained(A   )
									bert_tokenizer.save_pretrained(A   )
									lowercase_ : int							       =   CustomTokenizerFast.from_pretrained(A   )

							tokenizer.push_to_hub('''test-dynamic-tokenizer'''    ,						use_auth_token=self._token   )

							lowercase_ : str							       =   AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer'''    ,						trust_remote_code=A   )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizerFast'''   )
							lowercase_ : Union[str, Any]							       =   AutoTokenizer.from_pretrained(
							    F'''{USER}/test-dynamic-tokenizer'''    ,						use_fast=A    ,						trust_remote_code=A   )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__    ,						'''CustomTokenizer'''   )





class 					_UpperCAmelCase       ( unittest.TestCase							):




					def   A  (				self     :						Dict   )  ->  Any:
							lowercase_ : str							       =   Trie()
							trie.add('''Hello 友達'''   )
							self.assertEqual(trie.data    ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )
							trie.add('''Hello'''   )
							trie.data
							self.assertEqual(trie.data    ,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}   )




					def   A  (				self     :						Dict   )  ->  List[str]:
							lowercase_ : int							       =   Trie()
							self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )    ,						['''[CLS] This is a extra_id_100''']   )
							trie.add('''[CLS]'''   )
							trie.add('''extra_id_1'''   )
							trie.add('''extra_id_100'''   )
							self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''   )    ,						['''[CLS]''', ''' This is a ''', '''extra_id_100''']   )




					def   A  (				self     :						Any   )  ->  List[str]:
							lowercase_ : List[Any]							       =   Trie()
							trie.add('''A'''   )
							self.assertEqual(trie.split('''ABC'''   )    ,						['''A''', '''BC''']   )
							self.assertEqual(trie.split('''BCA'''   )    ,						['''BC''', '''A''']   )




					def   A  (				self     :						Optional[int]   )  ->  Optional[int]:
							lowercase_ : Dict							       =   Trie()
							trie.add('''TOKEN]'''   )
							trie.add('''[SPECIAL_TOKEN]'''   )
							self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )    ,						['''This is something ''', '''[SPECIAL_TOKEN]''']   )




					def   A  (				self     :						str   )  ->  Union[str, Any]:
							lowercase_ : Optional[int]							       =   Trie()
							trie.add('''A'''   )
							trie.add('''P'''   )
							trie.add('''[SPECIAL_TOKEN]'''   )
							self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''   )    ,						['''This is something ''', '''[SPECIAL_TOKEN]''']   )




					def   A  (				self     :						Optional[int]   )  ->  str:
							lowercase_ : Optional[Any]							       =   Trie()
							trie.add('''AB'''   )
							trie.add('''B'''   )
							trie.add('''C'''   )
							self.assertEqual(trie.split('''ABC'''   )    ,						['''AB''', '''C''']   )




					def   A  (				self     :						Optional[int]   )  ->  str:
							lowercase_ : Optional[Any]							       =   Trie()
							trie.add('''ABC'''   )
							trie.add('''B'''   )
							trie.add('''CD'''   )
							self.assertEqual(trie.split('''ABCD'''   )    ,						['''ABC''', '''D''']   )






					def   A  (				self     :						List[str]   )  ->  Optional[int]:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							lowercase_ : Union[str, Any]							       =   Trie()
							lowercase_ : Dict							       =   trie.cut_text('''ABC'''    ,						[0, 0, 2, 1, 2, 3]   )
							self.assertEqual(A    ,						['''AB''', '''C''']   )








"""simple docstring"""
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from argparse import ArgumentParser

from accelerate.commands.config import get_config_parser
from accelerate.commands.env import env_command_parser
from accelerate.commands.launch import launch_command_parser
from accelerate.commands.test import test_command_parser
from accelerate.commands.tpu import tpu_command_parser

def       _snake_case						(						):
 UpperCAmelCase	:					List[str]	=	ArgumentParser("""Accelerate CLI tool"""			,		usage="""accelerate <command> [<args>]"""			,		allow_abbrev=UpperCamelCase						)
 UpperCAmelCase	:					Dict	=	parser.add_subparsers(help="""accelerate command helpers"""						)

 # Register commands
 get_config_parser(subparsers=UpperCamelCase						)
 env_command_parser(subparsers=UpperCamelCase						)
 launch_command_parser(subparsers=UpperCamelCase						)
 tpu_command_parser(subparsers=UpperCamelCase						)
 test_command_parser(subparsers=UpperCamelCase						)

 # Let's go
 UpperCAmelCase	:					Optional[int]	=	parser.parse_args()

 if not hasattr(UpperCamelCase			,		"""func"""						):
  parser.print_help()
  exit(1						)

 # Run
 args.func(UpperCamelCase						)


if __name__ == "__main__":
      main()
