




import os
import unittest

from transformers.models.bartpho.tokenization_bartpho import VOCAB_FILES_NAMES, BartphoTokenizer
from transformers.testing_utils import get_tests_dir

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase	:		List[str]							       =   get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")


class 				UpperCamelCase_    ( UpperCAmelCase__		,					unittest.TestCase     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      BartphoTokenizer
    UpperCAmelCase__							  =      False
    UpperCAmelCase__							  =      True


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any])						->Union[str, Any]:





           '''simple docstring'''

           super().setUp()

           A__       =    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']
           A__       =    dict(zip(UpperCAmelCase__					, range(len(UpperCAmelCase__))))
           A__       =    {'''unk_token''': '''<unk>'''}

           A__       =    os.path.join(self.tmpdirname					, VOCAB_FILES_NAMES['''monolingual_vocab_file'''])
           with open(self.monolingual_vocab_file					, '''w'''					, encoding='''utf-8''') as fp:
                  for token in vocab_tokens:
                         fp.write(f"""{token} {vocab_tokens[token]}\n""")

           A__       =    BartphoTokenizer(UpperCAmelCase__					, self.monolingual_vocab_file					, **self.special_tokens_map)
           tokenizer.save_pretrained(self.tmpdirname)


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[int]					, **UpperCAmelCase__      :	Dict)						->Any:





           '''simple docstring'''

           kwargs.update(self.special_tokens_map)
           return BartphoTokenizer.from_pretrained(self.tmpdirname					, **UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Union[str, Any]					, UpperCAmelCase__      :	Optional[Any])						->Union[str, Any]:





           '''simple docstring'''

           A__       =    '''This is a là test'''
           A__       =    '''This is a<unk><unk> test'''
           return input_text, output_text





    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->int:





           '''simple docstring'''

           A__       =    BartphoTokenizer(UpperCAmelCase__					, self.monolingual_vocab_file					, **self.special_tokens_map)
           A__       =    '''This is a là test'''
           A__       =    '''▁This ▁is ▁a ▁l à ▁t est'''.split()
           A__       =    tokenizer.tokenize(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__					, UpperCAmelCase__)

           A__       =    tokens + [tokenizer.unk_token]
           A__       =    [4, 5, 6, 3, 3, 7, 8, 3]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__)					, UpperCAmelCase__)







'''simple docstring'''




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							return (torch.arange(state.num_processes		) + 1.0 + (state.num_processes * state.process_index)).to(state.device		)



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Any							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Union[str, Any]							     =		gather(_lowercase		)
							assert gathered_tensor.tolist() == list(range(1	,				state.num_processes**2 + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							UpperCAmelCase     :		Any							     =		[state.process_index]
							UpperCAmelCase     :		Union[str, Any]							     =		gather_object(_lowercase		)
							assert len(_lowercase		) == state.num_processes, F'''{gathered_obj}, {len(_lowercase		)} != {state.num_processes}'''
							assert gathered_obj == list(range(state.num_processes		)		), F'''{gathered_obj} != {list(range(state.num_processes		)		)}'''



def       __lowerCamelCase  (					_lowercase		)      ->			List[Any]:
							UpperCAmelCase     :		Optional[int]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		List[str]							     =		broadcast(_lowercase		)
							assert broadcasted_tensor.shape == torch.Size([state.num_processes]		)
							assert broadcasted_tensor.tolist() == list(range(1	,				state.num_processes + 1		)		)



def       __lowerCamelCase  (					_lowercase		)      ->			Tuple:
							# We need to pad the tensor with one more element if we are the main process
							# to ensure that we can pad
							if state.is_main_process:
														UpperCAmelCase     :		Optional[Any]							     =		torch.arange(state.num_processes + 1		).to(state.device		)
							else:
														UpperCAmelCase     :		Tuple							     =		torch.arange(state.num_processes		).to(state.device		)
							UpperCAmelCase     :		Optional[Any]							     =		pad_across_processes(_lowercase		)
							assert padded_tensor.shape == torch.Size([state.num_processes + 1]		)
							if not state.is_main_process:
														assert padded_tensor.tolist() == list(range(0	,				state.num_processes		)		) + [0]



def       __lowerCamelCase  (					_lowercase		)      ->			Dict:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Optional[Any]							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[Any]							     =		reduce(_lowercase	,				"""sum"""		)
							UpperCAmelCase     :		Optional[Any]							     =		torch.tensor([4.0, 6]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[Any]:
							# For now runs on only two processes
							if state.num_processes != 2:
														return
							UpperCAmelCase     :		Tuple							     =		create_tensor(_lowercase		)
							UpperCAmelCase     :		Optional[int]							     =		reduce(_lowercase	,				"""mean"""		)
							UpperCAmelCase     :		str							     =		torch.tensor([2.0, 3]		).to(state.device		)
							assert torch.allclose(_lowercase	,				_lowercase		), F'''{reduced_tensor} != {truth_tensor}'''



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							# For xla_spawn (TPUs)
							main()



def       __lowerCamelCase  (					)      ->			int:
							UpperCAmelCase     :		List[Any]							     =		PartialState()
							state.print(F'''State: {state}'''		)
							state.print("""testing gather"""		)
							test_gather(_lowercase		)
							state.print("""testing gather_object"""		)
							test_gather_object(_lowercase		)
							state.print("""testing broadcast"""		)
							test_broadcast(_lowercase		)
							state.print("""testing pad_across_processes"""		)
							test_pad_across_processes(_lowercase		)
							state.print("""testing reduce_sum"""		)
							test_reduce_sum(_lowercase		)
							state.print("""testing reduce_mean"""		)
							test_reduce_mean(_lowercase		)


if __name__ == "__main__":
							main()

