





"""simple docstring"""




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


a		:					List[str]	      =     logging.get_logger(__name__)

a		:					List[Any]	      =     {'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

a		:					Dict	      =     {
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

a		:					Tuple	      =     {
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       Optional[int]       ,			_lowercase		:       List[Any]   )					->Tuple:






	'''simple docstring'''






	with open(_lowercase       ,			"r"       ,			encoding="utf-8"   ) as f:
		a :						List[str]      =							json.loads(f.read()   )

	a :						str      =							collections.OrderedDict()
	a :						Any      =							collections.OrderedDict()
	a :						List[Any]      =							collections.OrderedDict()
	with open(_lowercase       ,			"r"       ,			encoding="utf-8"   ) as f:
		a :						int      =							f.readlines()
	a :						Any      =							[[t.rstrip("\n"   )] if (t == "," or "," not in t) else t.rstrip("\n"   ).split(","   ) for t in token]
	for idx, b in enumerate(_lowercase   ):
		a :						str      =							b
		a :						List[str]      =							idx
		for wd in b:
			a :						Any      =							idx

	return vocab, raw_vocab, ids_to_tokens, emoji


class 			__UpperCamelCase							(		a__		):
							lowerCamelCase  :   Any    =VOCAB_FILES_NAMES
							lowerCamelCase  :   Optional[Any]    =PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase  :   Tuple    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase  :   Any    =["""input_ids""", """attention_mask"""]






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__="<|endoftext|>" ,   lowerCAmelCase__="<|endoftext|>" ,   lowerCAmelCase__="<|startoftext|>" ,   lowerCAmelCase__="<|endoftext|>" ,   lowerCAmelCase__=False ,   **lowerCAmelCase__ ,   )      ->					Union[str, Any]:
								super().__init__(
								    unk_token=lowerCAmelCase__ ,   pad_token=lowerCAmelCase__ ,   bos_token=lowerCAmelCase__ ,   eos_token=lowerCAmelCase__ ,   do_clean_text=lowerCAmelCase__ ,   **lowerCAmelCase__ ,   )
								if not os.path.isfile(lowerCAmelCase__ ):
									raise ValueError(
									    f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
									    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" )
								if not os.path.isfile(lowerCAmelCase__ ):
									raise ValueError(
									    f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
									    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" )
								a :						Optional[int]      =							do_clean_text
								a,			a,			a,			a :						Tuple      =							load_vocab_and_emoji(lowerCAmelCase__ ,   lowerCAmelCase__ )
								a :						Any      =							SubWordJapaneseTokenizer(
								    vocab=self.vocab ,   ids_to_tokens=self.ids_to_tokens ,   emoji=self.emoji )






							@property
							def        __a  (  self )      ->					Any:
								# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
								return len(self.raw_vocab )






							def        __a  (  self )      ->					int:
								return dict(self.raw_vocab ,   **self.added_tokens_encoder )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					List[Any]:
								return self.subword_tokenizer.tokenize(lowerCAmelCase__ ,   clean=self.do_clean_text )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					List[Any]:
								return self.vocab.get(lowerCAmelCase__ ,   self.vocab.get(self.unk_token ) )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Union[str, Any]:
								return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					int:
								a :						Tuple      =							"".join(lowerCAmelCase__ ).strip()
								return out_string






							def        __a  (  self ,   lowerCAmelCase__ )      ->					List[int]:
								a :						Tuple      =							[]
								for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(lowerCAmelCase__ ,   add_special_tokens=lowerCAmelCase__ ) + [self.eos_token_id] )

								if len(lowerCAmelCase__ ) > self.model_max_length:
									a :						List[str]      =							input_ids[-self.model_max_length :]
								return input_ids






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					Tuple[str]:
								a :						List[Any]      =							0
								if os.path.isdir(lowerCAmelCase__ ):
									a :						int      =							os.path.join(
									    lowerCAmelCase__ ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )
									a :						Tuple      =							os.path.join(
									    lowerCAmelCase__ ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"] )
								else:
									a :						List[Any]      =							(
									    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
									)
									a :						Tuple      =							(
									    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
									)
								with open(lowerCAmelCase__ ,   "w" ,   encoding="utf-8" ) as writer:
									for token_index, token in self.ids_to_tokens.items():
										if index != token_index:
											logger.warning(
											    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
											    " Please check that the vocabulary is not corrupted!" )
											a :						int      =							token_index
										writer.write(",".join(lowerCAmelCase__ ) + "\n" )
										index += 1
								with open(lowerCAmelCase__ ,   "w" ,   encoding="utf-8" ) as writer:
									json.dump(self.emoji ,   lowerCAmelCase__ )
								return vocab_file, emoji_file


class 			__UpperCamelCase							(		a__		):






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Dict:
								a :						Union[str, Any]      =							vocab  # same as swe
								a :						str      =							ids_to_tokens  # same as bpe
								a :						str      =							emoji
								a :						Tuple      =							np.max([len(lowerCAmelCase__ ) for w in self.vocab.keys()] )
								a :						List[str]      =							re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)" )
								a :						Optional[int]      =							re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*" )
								a :						Dict      =							re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}" )
								a :						Tuple      =							re.compile(
								    R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*" )
								a :						str      =							re.compile(
								    R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*" )
								a :						Optional[int]      =							re.compile(
								    R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*" )
								a :						Optional[Any]      =							"─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
								a :						Union[str, Any]      =							"▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
								a :						Any      =							str.maketrans({k: "<BLOCK>" for k in keisen + blocks} )






							def __len__(  self )      ->					Dict:
								return len(self.ids_to_tokens )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Optional[int]:
								a :						List[Any]      =							self.content_repattera.sub("<URL>" ,   lowerCAmelCase__ )
								a :						Dict      =							self.content_repattera.sub("<EMAIL>" ,   lowerCAmelCase__ )
								a :						Tuple      =							self.content_repattera.sub("<TEL>" ,   lowerCAmelCase__ )
								a :						Union[str, Any]      =							self.content_repattera.sub("<DATE>" ,   lowerCAmelCase__ )
								a :						Optional[int]      =							self.content_repattera.sub("<DATE>" ,   lowerCAmelCase__ )
								a :						Any      =							self.content_repattera.sub("<PRICE>" ,   lowerCAmelCase__ )
								a :						Optional[int]      =							content.translate(self.content_transa )
								while "<BLOCK><BLOCK>" in content:
									a :						Any      =							content.replace("<BLOCK><BLOCK>" ,   "<BLOCK>" )
								return content






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=False )      ->					Optional[int]:
								a :						Optional[int]      =							text.replace(" " ,   "<SP>" )
								a :						int      =							text.replace("　" ,   "<SP>" )
								a :						Optional[Any]      =							text.replace("\r\n" ,   "<BR>" )
								a :						str      =							text.replace("\n" ,   "<BR>" )
								a :						str      =							text.replace("\r" ,   "<BR>" )
								a :						str      =							text.replace("\t" ,   "<TAB>" )
								a :						Any      =							text.replace("—" ,   "ー" )
								a :						List[str]      =							text.replace("−" ,   "ー" )
								for k, v in self.emoji["emoji"].items():
									if k in text:
										a :						int      =							text.replace(lowerCAmelCase__ ,   lowerCAmelCase__ )
								if clean:
									a :						Optional[int]      =							self.clean_text(lowerCAmelCase__ )

								def check_simbol(lowerCAmelCase__ ):
									a :						List[str]      =							x.encode()
									if len(lowerCAmelCase__ ) == 1 and len(lowerCAmelCase__ ) == 2:
										a :						Any      =							(int(e[0] ) << 8) + int(e[1] )
										if (
										    (c >= 0xc_2a1 and c <= 0xc_2bf)
										    or (c >= 0xc_780 and c <= 0xc_783)
										    or (c >= 0xc_ab9 and c <= 0xc_bbf)
										    or (c >= 0xc_c80 and c <= 0xc_da2)
										):
											return True
									return False

								def checkuae(lowerCAmelCase__ ):
									a :						str      =							x.encode()
									if len(lowerCAmelCase__ ) == 1 and len(lowerCAmelCase__ ) == 3:
										a :						Tuple      =							(int(e[0] ) << 16) + (int(e[1] ) << 8) + int(e[2] )
										if c >= 0xe28_080 and c <= 0xe2b_07f:
											return True
									return False

								a :						Optional[Any]      =							0
								a :						Tuple      =							[]
								while pos < len(lowerCAmelCase__ ):
									a :						Optional[Any]      =							min(len(lowerCAmelCase__ ) ,   pos + self.maxlen + 1 ) if text[pos] == "<" else pos + 3
									a :						Union[str, Any]      =							[]  # (token_id, token, pos)
									for e in range(lowerCAmelCase__ ,   lowerCAmelCase__ ,   -1 ):
										a :						List[Any]      =							text[pos:e]
										if wd in self.vocab:
											if wd[0] == "<" and len(lowerCAmelCase__ ) > 2:
												a :						Dict      =							[(self.vocab[wd], wd, e)]
												break
											else:
												candidates.append((self.vocab[wd], wd, e) )
									if len(lowerCAmelCase__ ) > 0:
										# the smallest token_id is adopted
										a,			a,			a :						List[Any]      =							sorted(lowerCAmelCase__ ,   key=lambda lowerCAmelCase__ : x[0] )[0]
										result.append(lowerCAmelCase__ )
										a :						Dict      =							e
									else:
										a :						Tuple      =							pos + 1
										a :						List[str]      =							text[pos:end]
										if check_simbol(lowerCAmelCase__ ):
											result.append("<KIGOU>" )
										elif checkuae(lowerCAmelCase__ ):
											result.append("<U2000U2BFF>" )
										else:
											for i in wd.encode("utf-8" ):
												result.append("<|byte%d|>" % i )
										a :						Tuple      =							end
								return result






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__="\n" )      ->					Any:
								a :						Dict      =							[]
								a :						List[str]      =							[]
								a :						List[str]      =							self.ids_to_tokens[index][0]
								if word[:6] == "<|byte" and word[-2:] == "|>":
									byte_tokens.append(int(word[6:-2] ) )
								else:
									if len(lowerCAmelCase__ ) > 0:
										words.append(bytearray(lowerCAmelCase__ ).decode("utf-8" ,   errors="replace" ) )
										a :						List[str]      =							[]
									if word[:7] == "<|emoji" and word[-2:] == "|>":
										words.append(self.emoji["emoji_inv"][word] )
									elif word == "<SP>":
										words.append(" " )
									elif word == "<BR>":
										words.append(lowerCAmelCase__ )
									elif word == "<TAB>":
										words.append("\t" )
									elif word == "<BLOCK>":
										words.append("▀" )
									elif word == "<KIGOU>":
										words.append("ǀ" )
									elif word == "<U2000U2BFF>":
										words.append("‖" )
									else:
										words.append(lowerCAmelCase__ )
								if len(lowerCAmelCase__ ) > 0:
									words.append(bytearray(lowerCAmelCase__ ).decode("utf-8" ,   errors="replace" ) )
								a :						Dict      =							"".join(lowerCAmelCase__ )
								return text









"""simple docstring"""




import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
		from transformers.models.esm.modeling_esm import (
		    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
		    EsmEmbeddings,
		    create_position_ids_from_input_ids,
		)


class 			__UpperCamelCase							:






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=13 ,   lowerCAmelCase__=7 ,   lowerCAmelCase__=False ,   lowerCAmelCase__=True ,   lowerCAmelCase__=False ,   lowerCAmelCase__=True ,   lowerCAmelCase__=33 ,   lowerCAmelCase__=32 ,   lowerCAmelCase__=5 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=37 ,   lowerCAmelCase__="gelu" ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=0.1 ,   lowerCAmelCase__=512 ,   lowerCAmelCase__=16 ,   lowerCAmelCase__=2 ,   lowerCAmelCase__=0.02 ,   lowerCAmelCase__=3 ,   lowerCAmelCase__=4 ,   lowerCAmelCase__=None ,   )      ->					List[str]:
								a :						List[str]      =							parent
								a :						Optional[int]      =							batch_size
								a :						Optional[int]      =							seq_length
								a :						Tuple      =							is_training
								a :						Optional[int]      =							use_input_mask
								a :						List[Any]      =							use_token_type_ids
								a :						Tuple      =							use_labels
								a :						Union[str, Any]      =							vocab_size
								a :						str      =							hidden_size
								a :						List[Any]      =							num_hidden_layers
								a :						Dict      =							num_attention_heads
								a :						int      =							intermediate_size
								a :						str      =							hidden_act
								a :						Optional[int]      =							hidden_dropout_prob
								a :						str      =							attention_probs_dropout_prob
								a :						str      =							max_position_embeddings
								a :						List[str]      =							type_vocab_size
								a :						List[str]      =							type_sequence_label_size
								a :						Union[str, Any]      =							initializer_range
								a :						int      =							num_labels
								a :						Optional[int]      =							num_choices
								a :						Union[str, Any]      =							scope






							def        __a  (  self )      ->					Optional[Any]:
								a :						str      =							ids_tensor([self.batch_size, self.seq_length] ,   self.vocab_size )

								a :						Dict      =							None
								if self.use_input_mask:
									a :						str      =							random_attention_mask([self.batch_size, self.seq_length] )

								a :						Union[str, Any]      =							None
								a :						int      =							None
								a :						int      =							None
								if self.use_labels:
									a :						Optional[int]      =							ids_tensor([self.batch_size] ,   self.type_sequence_label_size )
									a :						Optional[int]      =							ids_tensor([self.batch_size, self.seq_length] ,   self.num_labels )
									a :						List[Any]      =							ids_tensor([self.batch_size] ,   self.num_choices )

								a :						str      =							self.get_config()

								return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






							def        __a  (  self )      ->					Optional[int]:
								return EsmConfig(
								    vocab_size=self.vocab_size ,   hidden_size=self.hidden_size ,   pad_token_id=1 ,   num_hidden_layers=self.num_hidden_layers ,   num_attention_heads=self.num_attention_heads ,   intermediate_size=self.intermediate_size ,   hidden_act=self.hidden_act ,   hidden_dropout_prob=self.hidden_dropout_prob ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob ,   max_position_embeddings=self.max_position_embeddings ,   type_vocab_size=self.type_vocab_size ,   initializer_range=self.initializer_range ,   )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					str:
								a :						Union[str, Any]      =							EsmModel(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Optional[int]      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ )
								a :						List[str]      =							model(lowerCAmelCase__ )
								a :						Optional[Any]      =							model(lowerCAmelCase__ )

								self.parent.assertEqual(result.last_hidden_state.shape ,   (self.batch_size, self.seq_length, self.hidden_size) )
								self.parent.assertEqual(result.pooler_output.shape ,   (self.batch_size, self.hidden_size) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Tuple:
								a :						int      =							EsmForMaskedLM(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						Union[str, Any]      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   labels=lowerCAmelCase__ )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.vocab_size) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Optional[Any]:
								a :						str      =							self.num_labels
								a :						int      =							EsmForTokenClassification(config=lowerCAmelCase__ )
								model.to(lowerCAmelCase__ )
								model.eval()
								a :						int      =							model(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ,   labels=lowerCAmelCase__ )
								self.parent.assertEqual(result.logits.shape ,   (self.batch_size, self.seq_length, self.num_labels) )






							def        __a  (  self )      ->					Union[str, Any]:
								a :						str      =							self.prepare_config_and_inputs()
								(
								    (
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			(
								    a
								),			
								) :						Optional[int]      =							config_and_inputs
								a :						int      =							{"input_ids": input_ids, "attention_mask": input_mask}
								return config, inputs_dict


@require_torch
class 			__UpperCamelCase							(		a__						,			a__						,			unittest.TestCase		):
							lowerCamelCase  :   Optional[Any]    =False

							lowerCamelCase  :   Tuple    =(
							    (
							        EsmForMaskedLM,
							        EsmModel,
							        EsmForSequenceClassification,
							        EsmForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase  :   Tuple    =()
							lowerCamelCase  :   Optional[int]    =(
							    {
							        """feature-extraction""": EsmModel,
							        """fill-mask""": EsmForMaskedLM,
							        """text-classification""": EsmForSequenceClassification,
							        """token-classification""": EsmForTokenClassification,
							        """zero-shot""": EsmForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase  :   Any    =True






							def        __a  (  self )      ->					Tuple:
								a :						str      =							EsmModelTester(self )
								a :						Union[str, Any]      =							ConfigTester(self ,   config_class=lowerCAmelCase__ ,   hidden_size=37 )






							def        __a  (  self )      ->					Dict:
								self.config_tester.run_common_tests()






							def        __a  (  self )      ->					Union[str, Any]:
								a :						Tuple      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					List[str]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
									a :						List[str]      =							type
									self.model_tester.create_and_check_model(*lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[Any]:
								a :						Dict      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__ )






							def        __a  (  self )      ->					Dict:
								a :						int      =							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__ )






							@slow
							def        __a  (  self )      ->					Optional[Any]:
								for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									a :						Tuple      =							EsmModel.from_pretrained(lowerCAmelCase__ )
									self.assertIsNotNone(lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[int]:
								a :						List[str]      =							self.model_tester.prepare_config_and_inputs()[0]
								a :						str      =							EsmEmbeddings(config=lowerCAmelCase__ )

								a :						Optional[Any]      =							torch.as_tensor([[12, 31, 13, model.padding_idx]] )
								a :						Tuple      =							torch.as_tensor(
								    [
								        [
								            0 + model.padding_idx + 1,
								            1 + model.padding_idx + 1,
								            2 + model.padding_idx + 1,
								            model.padding_idx,
								        ]
								    ] )
								a :						str      =							create_position_ids_from_input_ids(lowerCAmelCase__ ,   model.padding_idx )
								self.assertEqual(position_ids.shape ,   expected_positions.shape )
								self.assertTrue(torch.all(torch.eq(lowerCAmelCase__ ,   lowerCAmelCase__ ) ) )






							def        __a  (  self )      ->					List[Any]:
								a :						int      =							self.model_tester.prepare_config_and_inputs()[0]
								a :						Dict      =							EsmEmbeddings(config=lowerCAmelCase__ )

								a :						Any      =							torch.empty(2 ,   4 ,   30 )
								a :						str      =							[
								    0 + embeddings.padding_idx + 1,
								    1 + embeddings.padding_idx + 1,
								    2 + embeddings.padding_idx + 1,
								    3 + embeddings.padding_idx + 1,
								]
								a :						Optional[Any]      =							torch.as_tensor([expected_single_positions, expected_single_positions] )
								a :						str      =							embeddings.create_position_ids_from_inputs_embeds(lowerCAmelCase__ )
								self.assertEqual(position_ids.shape ,   expected_positions.shape )
								self.assertTrue(torch.all(torch.eq(lowerCAmelCase__ ,   lowerCAmelCase__ ) ) )






							@unittest.skip("Esm does not support embedding resizing" )
							def        __a  (  self )      ->					Any:
								pass






							@unittest.skip("Esm does not support embedding resizing" )
							def        __a  (  self )      ->					Dict:
								pass






							@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests." )
							def        __a  (  self )      ->					str:
								pass


@require_torch
class 			__UpperCamelCase							(		a__		):






							@slow
							def        __a  (  self )      ->					int:
								with torch.no_grad():
									a :						str      =							EsmForMaskedLM.from_pretrained("facebook/esm2_t6_8M_UR50D" )
									model.eval()
									a :						List[str]      =							torch.tensor([[0, 1, 2, 3, 4, 5]] )
									a :						Any      =							model(lowerCAmelCase__ )[0]

									a :						Union[str, Any]      =							33

									a :						str      =							torch.Size((1, 6, vocab_size) )
									self.assertEqual(output.shape ,   lowerCAmelCase__ )

									a :						Optional[int]      =							torch.tensor(
									    [[[8.9_215, -10.5_898, -6.4_671], [-6.3_967, -13.9_114, -1.1_212], [-7.7_812, -13.9_516, -3.7_406]]] )
									self.assertTrue(torch.allclose(output[:, :3, :3] ,   lowerCAmelCase__ ,   atol=1E-4 ) )






							@slow
							def        __a  (  self )      ->					str:
								with torch.no_grad():
									a :						Optional[int]      =							EsmModel.from_pretrained("facebook/esm2_t6_8M_UR50D" )
									model.eval()

									a :						str      =							torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]] )
									a :						Tuple      =							model(lowerCAmelCase__ )[0]
									# compare the actual values for a slice.
									a :						Union[str, Any]      =							torch.tensor(
									    [[[0.1_444, 0.5_413, 0.3_248], [0.3_034, 0.0_053, 0.3_108], [0.3_228, -0.2_499, 0.3_415]]] )
									self.assertTrue(torch.allclose(output[:, :3, :3] ,   lowerCAmelCase__ ,   atol=1E-4 ) )


