




'''simple docstring'''







import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


lowerCAmelCase_     						=				logging.get_logger(__name__)

lowerCAmelCase_     						=				{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowerCAmelCase_     						=				{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowerCAmelCase_     						=				{
    '''abeja/gpt-neox-japanese-2.7b''': 2_0_4_8,
}







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   with open(a__      ,'r'      ,encoding='utf-8'						) as f:
      A__       =     json.loads(f.read()						)

   A__       =     collections.OrderedDict()
   A__       =     collections.OrderedDict()
   A__       =     collections.OrderedDict()
   with open(a__      ,'r'      ,encoding='utf-8'						) as f:
      A__       =     f.readlines()
   A__       =     [[t.rstrip('\n'						)] if (t == ',' or ',' not in t) else t.rstrip('\n'						).split(','						) for t in token]
   for idx, b in enumerate(a__						):
      A__       =     b
      A__       =     idx
      for wd in b:
         A__       =     idx

   return vocab, raw_vocab, ids_to_tokens, emoji

class 	_snake_case(			UpperCAmelCase      ):
 __snake_case:							Dict						=					VOCAB_FILES_NAMES
 __snake_case:							Optional[Any]						=					PRETRAINED_VOCAB_FILES_MAP
 __snake_case:							Union[str, Any]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:							Tuple						=					['input_ids', 'attention_mask']






 def __init__(self       :  Optional[int]    ,		a       :  Any    ,		a       :  List[Any]    ,		a       :  Dict="<|endoftext|>"    ,		a       :  List[Any]="<|endoftext|>"    ,		a       :  List[Any]="<|startoftext|>"    ,		a       :  int="<|endoftext|>"    ,		a       :  Any=False    ,		**a       :  str    ,		)       ->   List[Any]:






    """simple docstring"""


    super().__init__(
        unk_token=_SCREAMING_SNAKE_CASE    ,		pad_token=_SCREAMING_SNAKE_CASE    ,		bos_token=_SCREAMING_SNAKE_CASE    ,		eos_token=_SCREAMING_SNAKE_CASE    ,		do_clean_text=_SCREAMING_SNAKE_CASE    ,		**_SCREAMING_SNAKE_CASE    ,		)
    if not os.path.isfile(_SCREAMING_SNAKE_CASE    ):
       raise ValueError(
           f"""Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained"""
           ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'    )
    if not os.path.isfile(_SCREAMING_SNAKE_CASE    ):
       raise ValueError(
           f"""Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google"""
           ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'    )
    A__       =     do_clean_text
    A__  ,			A__  ,			A__  ,			A__       =     load_vocab_and_emoji(_SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     SubWordJapaneseTokenizer(
        vocab=self.vocab    ,		ids_to_tokens=self.ids_to_tokens    ,		emoji=self.emoji    )






 @property
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Tuple:






    """simple docstring"""


    return len(self.raw_vocab    )






 def 						_UpperCamelCase (self       :  List[str]    )       ->   Any:






    """simple docstring"""


    return dict(self.raw_vocab    ,		**self.added_tokens_encoder    )






 def 						_UpperCamelCase (self       :  Any    ,		a       :  Dict    )       ->   Union[str, Any]:






    """simple docstring"""


    return self.subword_tokenizer.tokenize(_SCREAMING_SNAKE_CASE    ,		clean=self.do_clean_text    )






 def 						_UpperCamelCase (self       :  Union[str, Any]    ,		a       :  Optional[Any]    )       ->   Union[str, Any]:






    """simple docstring"""


    return self.vocab.get(_SCREAMING_SNAKE_CASE    ,		self.vocab.get(self.unk_token    )    )






 def 						_UpperCamelCase (self       :  List[Any]    ,		a       :  Union[str, Any]    )       ->   int:






    """simple docstring"""


    return self.subword_tokenizer.convert_id_to_token(_SCREAMING_SNAKE_CASE    )






 def 						_UpperCamelCase (self       :  str    ,		a       :  List[str]    )       ->   List[Any]:






    """simple docstring"""


    A__       =     ''.join(_SCREAMING_SNAKE_CASE    ).strip()
    return out_string






 def 						_UpperCamelCase (self       :  Optional[int]    ,		a       :  Optional[int]    )       ->   List[int]:






    """simple docstring"""


    A__       =     []
    for is_user, text in conversation.iter_texts():
       input_ids.extend(self.encode(_SCREAMING_SNAKE_CASE    ,		add_special_tokens=_SCREAMING_SNAKE_CASE    ) + [self.eos_token_id]    )

    if len(_SCREAMING_SNAKE_CASE    ) > self.model_max_length:
       A__       =     input_ids[-self.model_max_length :]
    return input_ids






 def 						_UpperCamelCase (self       :  List[str]    ,		a       :  Tuple    ,		a       :  List[Any] = None    )       ->   Tuple[str]:






    """simple docstring"""


    A__       =     0
    if os.path.isdir(_SCREAMING_SNAKE_CASE    ):
       A__       =     os.path.join(
           _SCREAMING_SNAKE_CASE    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
       A__       =     os.path.join(
           _SCREAMING_SNAKE_CASE    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']    )
    else:
       A__       =     (
           (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
       )
       A__       =     (
           (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
       )
    with open(_SCREAMING_SNAKE_CASE    ,		'w'    ,		encoding='utf-8'    ) as writer:
       for token_index, token in self.ids_to_tokens.items():
          if index != token_index:
             logger.warning(
                 f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                 ' Please check that the vocabulary is not corrupted!'    )
             A__       =     token_index
          writer.write(','.join(_SCREAMING_SNAKE_CASE    ) + '\n'    )
          index += 1
    with open(_SCREAMING_SNAKE_CASE    ,		'w'    ,		encoding='utf-8'    ) as writer:
       json.dump(self.emoji    ,		_SCREAMING_SNAKE_CASE    )
    return vocab_file, emoji_file




class 	_snake_case(			UpperCAmelCase      ):






 def __init__(self       :  Any    ,		a       :  List[str]    ,		a       :  int    ,		a       :  Union[str, Any]    )       ->   Dict:






    """simple docstring"""


    A__       =     vocab  # same as swe
    A__       =     ids_to_tokens  # same as bpe
    A__       =     emoji
    A__       =     np.max([len(_SCREAMING_SNAKE_CASE    ) for w in self.vocab.keys()]    )
    A__       =     re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'    )
    A__       =     re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'    )
    A__       =     re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'    )
    A__       =     re.compile(
        R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'    )
    A__       =     re.compile(
        R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'    )
    A__       =     re.compile(
        R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'    )
    A__       =     '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
    A__       =     '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
    A__       =     str.maketrans({k: '<BLOCK>' for k in keisen + blocks}    )






 def __len__(self       :  int    )       ->   Optional[int]:






    """simple docstring"""


    return len(self.ids_to_tokens    )






 def 						_UpperCamelCase (self       :  Any    ,		a       :  List[str]    )       ->   Any:






    """simple docstring"""


    A__       =     self.content_repattera.sub('<URL>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     self.content_repattera.sub('<EMAIL>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     self.content_repattera.sub('<TEL>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     self.content_repattera.sub('<DATE>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     self.content_repattera.sub('<DATE>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     self.content_repattera.sub('<PRICE>'    ,		_SCREAMING_SNAKE_CASE    )
    A__       =     content.translate(self.content_transa    )
    while "<BLOCK><BLOCK>" in content:
       A__       =     content.replace('<BLOCK><BLOCK>'    ,		'<BLOCK>'    )
    return content






 def 						_UpperCamelCase (self       :  Optional[int]    ,		a       :  int    ,		a       :  Union[str, Any]=False    )       ->   int:






    """simple docstring"""


    A__       =     text.replace(' '    ,		'<SP>'    )
    A__       =     text.replace('　'    ,		'<SP>'    )
    A__       =     text.replace('\r\n'    ,		'<BR>'    )
    A__       =     text.replace('\n'    ,		'<BR>'    )
    A__       =     text.replace('\r'    ,		'<BR>'    )
    A__       =     text.replace('\t'    ,		'<TAB>'    )
    A__       =     text.replace('—'    ,		'ー'    )
    A__       =     text.replace('−'    ,		'ー'    )
    for k, v in self.emoji["emoji"].items():
       if k in text:
          A__       =     text.replace(_SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE    )
    if clean:
       A__       =     self.clean_text(_SCREAMING_SNAKE_CASE    )

    def check_simbol(a       :  Optional[int]    ):
       A__       =     x.encode()
       if len(_SCREAMING_SNAKE_CASE    ) == 1 and len(_SCREAMING_SNAKE_CASE    ) == 2:
          A__       =     (int(e[0]    ) << 8) + int(e[1]    )
          if (
              (c >= 0XC_2_A_1 and c <= 0XC_2_B_F)
              or (c >= 0XC_7_8_0 and c <= 0XC_7_8_3)
              or (c >= 0XC_A_B_9 and c <= 0XC_B_B_F)
              or (c >= 0XC_C_8_0 and c <= 0XC_D_A_2)
          ):
             return True
       return False

    def checkuae(a       :  Dict    ):
       A__       =     x.encode()
       if len(_SCREAMING_SNAKE_CASE    ) == 1 and len(_SCREAMING_SNAKE_CASE    ) == 3:
          A__       =     (int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
          if c >= 0XE_2_8_0_8_0 and c <= 0XE_2_B_0_7_F:
             return True
       return False

    A__       =     0
    A__       =     []
    while pos < len(_SCREAMING_SNAKE_CASE    ):
       A__       =     min(len(_SCREAMING_SNAKE_CASE    )    ,		pos + self.maxlen + 1    ) if text[pos] == '<' else pos + 3
       A__       =     []  # (token_id, token, pos)
       for e in range(_SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE    ,		-1    ):
          A__       =     text[pos:e]
          if wd in self.vocab:
             if wd[0] == "<" and len(_SCREAMING_SNAKE_CASE    ) > 2:
                A__       =     [(self.vocab[wd], wd, e)]
                break
             else:
                candidates.append((self.vocab[wd], wd, e)    )
       if len(_SCREAMING_SNAKE_CASE    ) > 0:
          # the smallest token_id is adopted
          A__  ,			A__  ,			A__       =     sorted(_SCREAMING_SNAKE_CASE    ,		key=lambda a    : x[0]    )[0]
          result.append(_SCREAMING_SNAKE_CASE    )
          A__       =     e
       else:
          A__       =     pos + 1
          A__       =     text[pos:end]
          if check_simbol(_SCREAMING_SNAKE_CASE    ):
             result.append('<KIGOU>'    )
          elif checkuae(_SCREAMING_SNAKE_CASE    ):
             result.append('<U2000U2BFF>'    )
          else:
             for i in wd.encode('utf-8'    ):
                result.append('<|byte%d|>' % i    )
          A__       =     end
    return result






 def 						_UpperCamelCase (self       :  Dict    ,		a       :  Tuple    ,		a       :  List[str]="\n"    )       ->   str:






    """simple docstring"""


    A__       =     []
    A__       =     []
    A__       =     self.ids_to_tokens[index][0]
    if word[:6] == "<|byte" and word[-2:] == "|>":
       byte_tokens.append(int(word[6:-2]    )    )
    else:
       if len(_SCREAMING_SNAKE_CASE    ) > 0:
          words.append(bytearray(_SCREAMING_SNAKE_CASE    ).decode('utf-8'    ,		errors='replace'    )    )
          A__       =     []
       if word[:7] == "<|emoji" and word[-2:] == "|>":
          words.append(self.emoji['emoji_inv'][word]    )
       elif word == "<SP>":
          words.append(' '    )
       elif word == "<BR>":
          words.append(_SCREAMING_SNAKE_CASE    )
       elif word == "<TAB>":
          words.append('\t'    )
       elif word == "<BLOCK>":
          words.append('▀'    )
       elif word == "<KIGOU>":
          words.append('ǀ'    )
       elif word == "<U2000U2BFF>":
          words.append('‖'    )
       else:
          words.append(_SCREAMING_SNAKE_CASE    )
    if len(_SCREAMING_SNAKE_CASE    ) > 0:
       words.append(bytearray(_SCREAMING_SNAKE_CASE    ).decode('utf-8'    ,		errors='replace'    )    )
    A__       =     ''.join(_SCREAMING_SNAKE_CASE    )
    return text






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY

@is_pipeline_test
class 						__a (							unittest.TestCase     ):
		_a      :    int           =      MODEL_FOR_MASKED_LM_MAPPING
		_a      :    Tuple           =      TF_MODEL_FOR_MASKED_LM_MAPPING
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									super().tearDown()
									# clean-up as much as possible GPU memory occupied by PyTorch
									gc.collect()
									if is_torch_available():
																import torch

																torch.cuda.empty_cache()
		@require_tf
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='sshleifer/tiny-distilroberta-base'     ,       top_k=2     ,       framework='tf'		)
									_UpperCAmelCase											=     unmasker('My name is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {'sequence': 'My name is grouped', 'score': 2.1e-0_5, 'token': 38015, 'token_str': ' grouped'},
									        {'sequence': 'My name is accuser', 'score': 2.1e-0_5, 'token': 25506, 'token_str': ' accuser'},
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('The largest city in France is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {
									            'sequence': 'The largest city in France is grouped',
									            'score': 2.1e-0_5,
									            'token': 38015,
									            'token_str': ' grouped',
									        },
									        {
									            'sequence': 'The largest city in France is accuser',
									            'score': 2.1e-0_5,
									            'token': 25506,
									            'token_str': ' accuser',
									        },
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('My name is <mask>'     ,       targets=[' Patrick', ' Clara', ' Teven']     ,       top_k=3		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {'sequence': 'My name is Clara', 'score': 2e-0_5, 'token': 13606, 'token_str': ' Clara'},
									        {'sequence': 'My name is Patrick', 'score': 2e-0_5, 'token': 3499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Te', 'score': 1.9e-0_5, 'token': 2941, 'token_str': ' Te'},
									    ]     ,       )
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='sshleifer/tiny-distilroberta-base'     ,       top_k=2     ,       framework='pt'		)

									_UpperCAmelCase											=     unmasker('My name is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {'sequence': 'My name is Maul', 'score': 2.2e-0_5, 'token': 35676, 'token_str': ' Maul'},
									        {'sequence': 'My name isELS', 'score': 2.2e-0_5, 'token': 16416, 'token_str': 'ELS'},
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('The largest city in France is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {
									            'sequence': 'The largest city in France is Maul',
									            'score': 2.2e-0_5,
									            'token': 35676,
									            'token_str': ' Maul',
									        },
									        {'sequence': 'The largest city in France isELS', 'score': 2.2e-0_5, 'token': 16416, 'token_str': 'ELS'},
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('My name is <mask>'     ,       targets=[' Patrick', ' Clara', ' Teven']     ,       top_k=3		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        {'sequence': 'My name is Patrick', 'score': 2.1e-0_5, 'token': 3499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Te', 'score': 2e-0_5, 'token': 2941, 'token_str': ' Te'},
									        {'sequence': 'My name is Clara', 'score': 2e-0_5, 'token': 13606, 'token_str': ' Clara'},
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('My name is <mask> <mask>'     ,       top_k=2		)

									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE     ,       decimals=6		)     ,       [
									        [
									            {
									                'score': 2.2e-0_5,
									                'token': 35676,
									                'token_str': ' Maul',
									                'sequence': '<s>My name is Maul<mask></s>',
									            },
									            {'score': 2.2e-0_5, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
									        ],
									        [
									            {
									                'score': 2.2e-0_5,
									                'token': 35676,
									                'token_str': ' Maul',
									                'sequence': '<s>My name is<mask> Maul</s>',
									            },
									            {'score': 2.2e-0_5, 'token': 16416, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
									        ],
									    ]     ,       )
		@require_torch_gpu
		def       UpperCAmelCase__							(     self		)					->			List[Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline('fill-mask'     ,       model='hf-internal-testing/tiny-random-distilbert'     ,       device=0     ,       framework='pt'		)

									# convert model to fp16
									pipe.model.half()

									_UpperCAmelCase											=     pipe('Paris is the [MASK] of France.'		)
									# We actually don't care about the result, we just want to make sure
									# it works, meaning the float16 tensor got casted back to float32
									# for postprocessing.
									self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
		@slow
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='distilroberta-base'     ,       top_k=2     ,       framework='pt'		)
									self.run_large_test(_SCREAMING_SNAKE_CASE		)
		@slow
		@require_tf
		def       UpperCAmelCase__							(     self		)					->			Dict:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='distilroberta-base'     ,       top_k=2     ,       framework='tf'		)
									self.run_large_test(_SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     unmasker('My name is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       [
									        {'sequence': 'My name is John', 'score': 0.008, 'token': 610, 'token_str': ' John'},
									        {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1573, 'token_str': ' Chris'},
									    ]     ,       )
									_UpperCAmelCase											=     unmasker('The largest city in France is <mask>'		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       [
									        {
									            'sequence': 'The largest city in France is Paris',
									            'score': 0.251,
									            'token': 2201,
									            'token_str': ' Paris',
									        },
									        {
									            'sequence': 'The largest city in France is Lyon',
									            'score': 0.214,
									            'token': 12790,
									            'token_str': ' Lyon',
									        },
									    ]     ,       )

									_UpperCAmelCase											=     unmasker('My name is <mask>'     ,       targets=[' Patrick', ' Clara', ' Teven']     ,       top_k=3		)
									self.assertEqual(
									    nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       [
									        {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3499, 'token_str': ' Patrick'},
									        {'sequence': 'My name is Clara', 'score': 0.000, 'token': 13606, 'token_str': ' Clara'},
									        {'sequence': 'My name is Te', 'score': 0.000, 'token': 2941, 'token_str': ' Te'},
									    ]     ,       )
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			Dict:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='sshleifer/tiny-distilroberta-base'     ,       framework='pt'		)
									_UpperCAmelCase											=     None
									_UpperCAmelCase											=     None
									self.run_pipeline_test(_SCREAMING_SNAKE_CASE     ,       []		)
		@require_tf
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									_UpperCAmelCase											=     pipeline(task='fill-mask'     ,       model='sshleifer/tiny-distilroberta-base'     ,       framework='tf'		)
									_UpperCAmelCase											=     None
									_UpperCAmelCase											=     None
									self.run_pipeline_test(_SCREAMING_SNAKE_CASE     ,       []		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			List[Any]:




									"""simple docstring"""





									if tokenizer is None or tokenizer.mask_token_id is None:
																self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'		)

									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     [
									    f'''This is another {tokenizer.mask_token} test''',
									]
									return fill_masker, examples
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			List[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     fill_masker.tokenizer
									_UpperCAmelCase											=     fill_masker.model

									_UpperCAmelCase											=     fill_masker(
									    f'''This is a {tokenizer.mask_token}'''     ,       )
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )

									_UpperCAmelCase											=     fill_masker([f'''This is a {tokenizer.mask_token}''']		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )

									_UpperCAmelCase											=     fill_masker([f'''This is a {tokenizer.mask_token}''', f'''Another {tokenizer.mask_token} great test.''']		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        [
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        ],
									        [
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        ],
									    ]     ,       )

									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																fill_masker([None]		)
									# No mask_token is not supported
									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																fill_masker('This is'		)

									self.run_test_top_k(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
									self.run_test_targets(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
									self.run_test_top_k_targets(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
									self.fill_mask_with_duplicate_targets_and_top_k(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
									self.fill_mask_with_multiple_masks(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     tokenizer.get_vocab()
									_UpperCAmelCase											=     sorted(vocab.keys()		)[:2]
									# Pipeline argument
									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE     ,       targets=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )
									_UpperCAmelCase											=     {vocab[el] for el in targets}
									self.assertEqual({el['token'] for el in outputs}     ,       _SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     [tokenizer.decode([x]		) for x in target_ids]
									self.assertEqual({el['token_str'] for el in outputs}     ,       set(_SCREAMING_SNAKE_CASE		)		)

									# Call argument
									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=_SCREAMING_SNAKE_CASE		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )
									_UpperCAmelCase											=     {vocab[el] for el in targets}
									self.assertEqual({el['token'] for el in outputs}     ,       _SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     [tokenizer.decode([x]		) for x in target_ids]
									self.assertEqual({el['token_str'] for el in outputs}     ,       set(_SCREAMING_SNAKE_CASE		)		)

									# Score equivalence
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     [top_mask['token_str'] for top_mask in outputs]
									_UpperCAmelCase											=     [top_mask['score'] for top_mask in outputs]

									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(_SCREAMING_SNAKE_CASE		) == set(_SCREAMING_SNAKE_CASE		):
																_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=_SCREAMING_SNAKE_CASE		)
																_UpperCAmelCase											=     [top_mask['score'] for top_mask in unmasked_targets]
																self.assertEqual(nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       nested_simplify(_SCREAMING_SNAKE_CASE		)		)

									# Raises with invalid
									with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=[]		)
									# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
									if "" not in tokenizer.get_vocab():
																with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																							_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=['']		)
																with self.assertRaises(_SCREAMING_SNAKE_CASE		):
																							_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       targets=''		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE     ,       top_k=2		)
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )

									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       top_k=2		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									    ]     ,       )
									self.assertEqual(nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       nested_simplify(_SCREAMING_SNAKE_CASE		)		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			int:




									"""simple docstring"""





									_UpperCAmelCase											=     tokenizer.get_vocab()
									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)

									# top_k=2, ntargets=3
									_UpperCAmelCase											=     sorted(vocab.keys()		)[:3]
									_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       top_k=2     ,       targets=_SCREAMING_SNAKE_CASE		)

									# If we use the most probably targets, and filter differently, we should still
									# have the same results
									_UpperCAmelCase											=     [el['token_str'] for el in sorted(_SCREAMING_SNAKE_CASE     ,       key=lambda _SCREAMING_SNAKE_CASE		: x["score"]     ,       reverse=_SCREAMING_SNAKE_CASE		)]
									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(_SCREAMING_SNAKE_CASE		).issubset(_SCREAMING_SNAKE_CASE		):
																_UpperCAmelCase											=     fill_masker(f'''This is a {tokenizer.mask_token}'''     ,       top_k=3     ,       targets=_SCREAMING_SNAKE_CASE		)
																# They should yield exactly the same result
																self.assertEqual(nested_simplify(_SCREAMING_SNAKE_CASE		)     ,       nested_simplify(_SCREAMING_SNAKE_CASE		)		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			str:




									"""simple docstring"""





									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     tokenizer.get_vocab()
									# String duplicates + id duplicates
									_UpperCAmelCase											=     sorted(vocab.keys()		)[:3]
									_UpperCAmelCase											=     [targets[0], targets[1], targets[0], targets[2], targets[1]]
									_UpperCAmelCase											=     fill_masker(f'''My name is {tokenizer.mask_token}'''     ,       targets=_SCREAMING_SNAKE_CASE     ,       top_k=10		)

									# The target list contains duplicates, so we can't output more
									# than them
									self.assertEqual(len(_SCREAMING_SNAKE_CASE		)     ,       3		)






		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE		)					->			Tuple:




									"""simple docstring"""





									_UpperCAmelCase											=     FillMaskPipeline(model=_SCREAMING_SNAKE_CASE     ,       tokenizer=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     fill_masker(
									    f'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''     ,       top_k=2		)
									self.assertEqual(
									    _SCREAMING_SNAKE_CASE     ,       [
									        [
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        ],
									        [
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        ],
									        [
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									            {'sequence': ANY(_SCREAMING_SNAKE_CASE		), 'score': ANY(_SCREAMING_SNAKE_CASE		), 'token': ANY(_SCREAMING_SNAKE_CASE		), 'token_str': ANY(_SCREAMING_SNAKE_CASE		)},
									        ],
									    ]     ,       )
