



"""simple docstring"""
import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel
class 							_UpperCAmelCase							:





     '''simple docstring'''



     def __init__(					self       ,	A       ,	A=1_3       ,	A=7       ,	A=True       ,	A=True       ,	A=True       ,	A=True       ,	A=9_9       ,	A=3_2       ,	A=5       ,	A=4       ,	A=4       ,	A="gelu"       ,	A=0.0       ,	A=0.1       ,	A=True       ,	A=5_1_2       ,	A=1_6       ,	A=2       ,	A=0.02       ,	A=3       ,	A=4       ,	A=None       ,	)							->					List[str]:
         _UpperCAmelCase							:  Optional[Any]											=    parent
         _UpperCAmelCase							:  Any											=    batch_size
         _UpperCAmelCase							:  Any											=    seq_length
         _UpperCAmelCase							:  List[Any]											=    is_training
         _UpperCAmelCase							:  List[str]											=    use_input_mask
         _UpperCAmelCase							:  List[Any]											=    use_token_type_ids
         _UpperCAmelCase							:  List[Any]											=    use_labels
         _UpperCAmelCase							:  Optional[int]											=    vocab_size
         _UpperCAmelCase							:  str											=    hidden_size
         _UpperCAmelCase							:  Dict											=    num_hidden_layers
         _UpperCAmelCase							:  str											=    num_attention_heads
         _UpperCAmelCase							:  List[str]											=    intermediate_multiple_size
         _UpperCAmelCase							:  Optional[int]											=    hidden_act
         _UpperCAmelCase							:  Optional[int]											=    hidden_dropout
         _UpperCAmelCase							:  Dict											=    attention_dropout
         _UpperCAmelCase							:  List[str]											=    weight_tying
         _UpperCAmelCase							:  Any											=    max_position_embeddings
         _UpperCAmelCase							:  Optional[int]											=    type_vocab_size
         _UpperCAmelCase							:  Tuple											=    type_sequence_label_size
         _UpperCAmelCase							:  Optional[Any]											=    initializer_range
         _UpperCAmelCase							:  Optional[int]											=    num_labels
         _UpperCAmelCase							:  Dict											=    num_choices
         _UpperCAmelCase							:  List[str]											=    scope





     def  __lowerCAmelCase	(					self		)							->					List[str]:
         _UpperCAmelCase							:  int											=    ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size		)

         _UpperCAmelCase							:  List[Any]											=    None
         if self.use_input_mask:
             _UpperCAmelCase							:  List[str]											=    random_attention_mask([self.batch_size, self.seq_length]		)

         _UpperCAmelCase							:  Dict											=    None
         if self.use_labels:
             _UpperCAmelCase							:  Union[str, Any]											=    ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels		)

         _UpperCAmelCase							:  Any											=    self.get_config()

         return config, input_ids, input_mask, token_labels





     def  __lowerCAmelCase	(					self		)							->					Union[str, Any]:
         return GPTNeoXJapaneseConfig(
             vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_multiple_size=self.intermediate_multiple_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout=self.hidden_dropout       ,	attention_dropout=self.attention_dropout       ,	weight_tying=self.weight_tying       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	is_decoder=A       ,	initializer_range=self.initializer_range       ,	)





     def  __lowerCAmelCase	(					self		)							->					Dict:
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  str											=    self.prepare_config_and_inputs()

         _UpperCAmelCase							:  str											=    True

         return config, input_ids, input_mask, token_labels





     def  __lowerCAmelCase	(					self       ,	A       ,	A       ,	A		)							->					int:
         _UpperCAmelCase							:  Optional[Any]											=    GPTNeoXJapaneseModel(config=A		)
         model.to(A		)
         model.eval()
         _UpperCAmelCase							:  Dict											=    model(A       ,	attention_mask=A		)
         _UpperCAmelCase							:  Union[str, Any]											=    model(A		)
         self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)		)





     def  __lowerCAmelCase	(					self       ,	A       ,	A       ,	A		)							->					Optional[Any]:
         _UpperCAmelCase							:  List[Any]											=    True
         _UpperCAmelCase							:  Any											=    GPTNeoXJapaneseModel(A		)
         model.to(A		)
         model.eval()
         _UpperCAmelCase							:  List[Any]											=    model(A       ,	attention_mask=A		)
         self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)		)





     def  __lowerCAmelCase	(					self       ,	A       ,	A       ,	A       ,	A		)							->					str:
         _UpperCAmelCase							:  str											=    GPTNeoXJapaneseForCausalLM(config=A		)
         model.to(A		)
         model.eval()
         _UpperCAmelCase							:  Any											=    model(A       ,	attention_mask=A       ,	labels=A		)
         self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)		)





     def  __lowerCAmelCase	(					self       ,	A       ,	A       ,	A		)							->					str:
         _UpperCAmelCase							:  Tuple											=    True
         _UpperCAmelCase							:  Dict											=    GPTNeoXJapaneseForCausalLM(config=A		)
         model.to(A		)
         model.eval()

         # first forward pass
         _UpperCAmelCase							:  Any											=    model(A       ,	attention_mask=A       ,	use_cache=A		)
         _UpperCAmelCase							:  Tuple											=    outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         _UpperCAmelCase							:  int											=    ids_tensor((self.batch_size, 3)       ,	config.vocab_size		)
         _UpperCAmelCase							:  List[Any]											=    ids_tensor((self.batch_size, 3)       ,	vocab_size=2		)

         # append to next input_ids and
         _UpperCAmelCase							:  List[str]											=    torch.cat([input_ids, next_tokens]       ,	dim=-1		)
         _UpperCAmelCase							:  Dict											=    torch.cat([input_mask, next_mask]       ,	dim=-1		)

         _UpperCAmelCase							:  Dict											=    model(A       ,	attention_mask=A       ,	output_hidden_states=A		)
         _UpperCAmelCase							:  Tuple											=    output_from_no_past['''hidden_states'''][0]
         _UpperCAmelCase							:  Optional[int]											=    model(
             A       ,	attention_mask=A       ,	past_key_values=A       ,	output_hidden_states=A       ,	)['''hidden_states'''][0]

         # select random slice
         _UpperCAmelCase							:  Optional[int]											=    ids_tensor((1,)       ,	output_from_past.shape[-1]		).item()
         _UpperCAmelCase							:  Union[str, Any]											=    output_from_no_past[:, -3:, random_slice_idx].detach()
         _UpperCAmelCase							:  Optional[Any]											=    output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(A       ,	A       ,	atol=1E-3		)		)





     def  __lowerCAmelCase	(					self		)							->					Dict:
         _UpperCAmelCase							:  Union[str, Any]											=    self.prepare_config_and_inputs()
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  Dict											=    config_and_inputs
         _UpperCAmelCase							:  int											=    {'''input_ids''': input_ids, '''attention_mask''': input_mask}
         return config, inputs_dict



@require_torch
class 							_UpperCAmelCase							( a		,a		,unittest.TestCase      ):





     '''simple docstring'''



     a__									=(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
     a__									=(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
     a__									=(
         {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
         if is_torch_available()
         else {}
     )
     a__									=False
     a__									=False
     a__									=False
     a__									=False





     def  __lowerCAmelCase	(					self		)							->					Any:
         _UpperCAmelCase							:  Tuple											=    GPTNeoXJapaneseModelTester(self		)
         _UpperCAmelCase							:  Tuple											=    ConfigTester(self       ,	config_class=A       ,	hidden_size=3_7		)





     def  __lowerCAmelCase	(					self		)							->					str:
         self.config_tester.run_common_tests()





     def  __lowerCAmelCase	(					self		)							->					Tuple:
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  List[str]											=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(A       ,	A       ,	A		)





     def  __lowerCAmelCase	(					self		)							->					Tuple:
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  List[Any]											=    self.model_tester.prepare_config_and_inputs_for_decoder()
         self.model_tester.create_and_check_model_as_decoder(A       ,	A       ,	A		)





     def  __lowerCAmelCase	(					self		)							->					str:
         # This regression test was failing with PyTorch < 1.3
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  Any											=    self.model_tester.prepare_config_and_inputs_for_decoder()

         _UpperCAmelCase							:  Union[str, Any]											=    None

         self.model_tester.create_and_check_model_as_decoder(A       ,	A       ,	A		)





     def  __lowerCAmelCase	(					self		)							->					Union[str, Any]:
         _UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							,				_UpperCAmelCase							:  Optional[Any]											=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_decoder_model_past_large_inputs(A       ,	A       ,	A		)





     def  __lowerCAmelCase	(					self		)							->					str:
         _UpperCAmelCase							:  Optional[int]											=    self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_causal_lm(*A		)





     @slow
     def  __lowerCAmelCase	(					self		)							->					int:
         _UpperCAmelCase							:  List[Any]											=    '''abeja/gpt-neox-japanese-2.7b'''

         _UpperCAmelCase							:  str											=    ['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

         _UpperCAmelCase							:  Any											=    [
             '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
             '''100年後に必要とされる会社は、「人」が中心の会社です。''',
             '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
             '''国境の長いトンネルを抜けると、そこは雪国だった。''',
             '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
         ]

         _UpperCAmelCase							:  str											=    GPTNeoXJapaneseTokenizer.from_pretrained(A		)
         _UpperCAmelCase							:  Any											=    GPTNeoXJapaneseForCausalLM.from_pretrained(A		)

         _UpperCAmelCase							:  Tuple											=    []
         for prompt in prompts:
             _UpperCAmelCase							:  str											=    tokenizer(A       ,	return_tensors='''pt'''		).input_ids
             _UpperCAmelCase							:  List[Any]											=    model.generate(A       ,	max_length=5_0		)
             _UpperCAmelCase							:  Union[str, Any]											=    tokenizer.batch_decode(A       ,	skip_special_tokens=A		)
             predicted_outputs += generated_string
         self.assertListEqual(A       ,	A		)







"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase    :Optional[int]     =					logging.get_logger(__name__)

_lowerCAmelCase    :List[str]     =					'▁'

_lowerCAmelCase    :Tuple     =					{'vocab_file': 'sentencepiece.bpe.model'}

_lowerCAmelCase    :List[Any]     =					{
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

_lowerCAmelCase    :Tuple     =					{
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}
class 							_UpperCAmelCase							( a      ):





     '''simple docstring'''



     a__									=VOCAB_FILES_NAMES
     a__									=PRETRAINED_VOCAB_FILES_MAP
     a__									=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     a__									=['''input_ids''', '''attention_mask''']





     def __init__(					self       ,	A       ,	A="<s>"       ,	A="</s>"       ,	A="</s>"       ,	A="<s>"       ,	A="<unk>"       ,	A="<pad>"       ,	A="<mask>"       ,	A = None       ,	**A       ,	)							->					None:
         # Mask token behave like a normal word, i.e. include the space before it
         _UpperCAmelCase							:  Tuple											=    AddedToken(A       ,	lstrip=A       ,	rstrip=A		) if isinstance(A       ,	A		) else mask_token

         _UpperCAmelCase							:  List[Any]											=    {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=A       ,	eos_token=A       ,	unk_token=A       ,	sep_token=A       ,	cls_token=A       ,	pad_token=A       ,	mask_token=A       ,	sp_model_kwargs=self.sp_model_kwargs       ,	**A       ,	)

         _UpperCAmelCase							:  Any											=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.Load(str(A		)		)
         _UpperCAmelCase							:  List[Any]											=    vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         _UpperCAmelCase							:  List[str]											=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         _UpperCAmelCase							:  Any											=    1

         _UpperCAmelCase							:  Optional[Any]											=    len(self.sp_model		) + self.fairseq_offset
         _UpperCAmelCase							:  int											=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}





     def __getstate__(					self		)							->					Union[str, Any]:
         _UpperCAmelCase							:  Tuple											=    self.__dict__.copy()
         _UpperCAmelCase							:  List[str]											=    None
         _UpperCAmelCase							:  str											=    self.sp_model.serialized_model_proto()
         return state





     def __setstate__(					self       ,	A		)							->					Optional[int]:
         _UpperCAmelCase							:  Optional[int]											=    d

         # for backward compatibility
         if not hasattr(self       ,	'''sp_model_kwargs'''		):
             _UpperCAmelCase							:  Optional[Any]											=    {}

         _UpperCAmelCase							:  Optional[int]											=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)





     def  __lowerCAmelCase	(					self       ,	A       ,	A = None		)							->					List[int]:

         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _UpperCAmelCase							:  Any											=    [self.cls_token_id]
         _UpperCAmelCase							:  Any											=    [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep





     def  __lowerCAmelCase	(					self       ,	A       ,	A = None       ,	A = False		)							->					List[int]:

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=A       ,	token_ids_a=A       ,	already_has_special_tokens=A		)

         if token_ids_a is None:
             return [1] + ([0] * len(A		)) + [1]
         return [1] + ([0] * len(A		)) + [1, 1] + ([0] * len(A		)) + [1]





     def  __lowerCAmelCase	(					self       ,	A       ,	A = None		)							->					List[int]:
         _UpperCAmelCase							:  Dict											=    [self.sep_token_id]
         _UpperCAmelCase							:  List[str]											=    [self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]





     @property
     def  __lowerCAmelCase	(					self		)							->					Dict:
         return len(self.sp_model		) + self.fairseq_offset + 1  # Add the <mask> token





     def  __lowerCAmelCase	(					self		)							->					Tuple:
         _UpperCAmelCase							:  Dict											=    {self.convert_ids_to_tokens(A		): i for i in range(self.vocab_size		)}
         vocab.update(self.added_tokens_encoder		)
         return vocab





     def  __lowerCAmelCase	(					self       ,	A		)							->					List[str]:
         return self.sp_model.encode(A       ,	out_type=A		)





     def  __lowerCAmelCase	(					self       ,	A		)							->					Any:
         if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
         _UpperCAmelCase							:  Any											=    self.sp_model.PieceToId(A		)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





     def  __lowerCAmelCase	(					self       ,	A		)							->					int:
         if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset		)





     def  __lowerCAmelCase	(					self       ,	A		)							->					int:
         _UpperCAmelCase							:  str											=    ''''''.join(A		).replace(A       ,	''' '''		).strip()
         return out_string





     def  __lowerCAmelCase	(					self       ,	A       ,	A = None		)							->					Tuple[str]:
         if not os.path.isdir(A		):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
             return
         _UpperCAmelCase							:  List[Any]											=    os.path.join(
             A       ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

         if os.path.abspath(self.vocab_file		) != os.path.abspath(A		) and os.path.isfile(self.vocab_file		):
             copyfile(self.vocab_file       ,	A		)
         elif not os.path.isfile(self.vocab_file		):
             with open(A       ,	'''wb'''		) as fi:
                 _UpperCAmelCase							:  str											=    self.sp_model.serialized_model_proto()
                 fi.write(A		)

         return (out_vocab_file,)


