





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


_snake_case    :       Optional[Any]		       =    logging.get_logger(__name__)

_snake_case    :       Optional[int]		       =    {'vocab_file': 'vocab.txt', 'emoji_file': 'emoji.json'}

_snake_case    :       Dict		       =    {
    'vocab_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt',
    },
    'emoji_file': {
        'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json',
    },
}

_snake_case    :       str		       =    {
    'abeja/gpt-neox-japanese-2.7b': 2048,
}

def     a_       (					lowerCAmelCase_ :     Optional[Any],      lowerCAmelCase_ :     List[str]      ):
   with open(lowerCAmelCase_,      'r',      encoding='utf-8'      ) as f:
      __lowerCAmelCase				      =      json.loads(f.read()      )

   __lowerCAmelCase				      =      collections.OrderedDict()
   __lowerCAmelCase				      =      collections.OrderedDict()
   __lowerCAmelCase				      =      collections.OrderedDict()
   with open(lowerCAmelCase_,      'r',      encoding='utf-8'      ) as f:
      __lowerCAmelCase				      =      f.readlines()
   __lowerCAmelCase				      =      [[t.rstrip('\n'      )] if (t == ',' or ',' not in t) else t.rstrip('\n'      ).split(','      ) for t in token]
   for idx, b in enumerate(lowerCAmelCase_      ):
      __lowerCAmelCase				      =      b
      __lowerCAmelCase				      =      idx
      for wd in b:
         __lowerCAmelCase				      =      idx

   return vocab, raw_vocab, ids_to_tokens, emoji



class 							_UpperCAmelCase       (       _UpperCamelCase       ):

  """simple docstring"""


  a_              =					VOCAB_FILES_NAMES
  a_              =					PRETRAINED_VOCAB_FILES_MAP
  a_              =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_              =					["""input_ids""", """attention_mask"""]






  def __init__(							self					:						Tuple   ,    lowerCAmelCase_					:						Optional[Any]   ,    lowerCAmelCase_					:						Optional[Any]   ,    lowerCAmelCase_					:						Tuple="<|endoftext|>"   ,    lowerCAmelCase_					:						Optional[int]="<|endoftext|>"   ,    lowerCAmelCase_					:						Optional[Any]="<|startoftext|>"   ,    lowerCAmelCase_					:						Optional[Any]="<|endoftext|>"   ,    lowerCAmelCase_					:						Union[str, Any]=False   ,    **lowerCAmelCase_					:						List[str]   ,    )				->		Any:
     super().__init__(
         unk_token=lowerCAmelCase_   ,    pad_token=lowerCAmelCase_   ,    bos_token=lowerCAmelCase_   ,    eos_token=lowerCAmelCase_   ,    do_clean_text=lowerCAmelCase_   ,    **lowerCAmelCase_   ,    )
     if not os.path.isfile(lowerCAmelCase_	):
        raise ValueError(
            f"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
            ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'	)
     if not os.path.isfile(lowerCAmelCase_	):
        raise ValueError(
            f"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
            ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'	)
     __lowerCAmelCase				      =      do_clean_text
     __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase				      =      load_vocab_and_emoji(lowerCAmelCase_   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      SubWordJapaneseTokenizer(
         vocab=self.vocab   ,    ids_to_tokens=self.ids_to_tokens   ,    emoji=self.emoji	)






  @property
  def 							lowercase						(							self					:						List[Any]	)				->		List[Any]:
     # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
     return len(self.raw_vocab	)






  def 							lowercase						(							self					:						Any	)				->		Tuple:
     return dict(self.raw_vocab   ,    **self.added_tokens_encoder	)






  def 							lowercase						(							self					:						Any   ,    lowerCAmelCase_					:						Optional[Any]	)				->		Tuple:
     return self.subword_tokenizer.tokenize(lowerCAmelCase_   ,    clean=self.do_clean_text	)






  def 							lowercase						(							self					:						Optional[int]   ,    lowerCAmelCase_					:						Dict	)				->		Optional[int]:
     return self.vocab.get(lowerCAmelCase_   ,    self.vocab.get(self.unk_token	)	)






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						int	)				->		int:
     return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase_	)






  def 							lowercase						(							self					:						Optional[Any]   ,    lowerCAmelCase_					:						Dict	)				->		Optional[int]:
     __lowerCAmelCase				      =      ''.join(lowerCAmelCase_	).strip()
     return out_string






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						"Conversation"	)				->		List[int]:
     __lowerCAmelCase				      =      []
     for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(lowerCAmelCase_   ,    add_special_tokens=lowerCAmelCase_	) + [self.eos_token_id]	)

     if len(lowerCAmelCase_	) > self.model_max_length:
        __lowerCAmelCase				      =      input_ids[-self.model_max_length :]
     return input_ids






  def 							lowercase						(							self					:						List[Any]   ,    lowerCAmelCase_					:						str   ,    lowerCAmelCase_					:						Optional[str] = None	)				->		Tuple[str]:
     __lowerCAmelCase				      =      0
     if os.path.isdir(lowerCAmelCase_	):
        __lowerCAmelCase				      =      os.path.join(
            lowerCAmelCase_   ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
        __lowerCAmelCase				      =      os.path.join(
            lowerCAmelCase_   ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']	)
     else:
        __lowerCAmelCase				      =      (
            (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
        )
        __lowerCAmelCase				      =      (
            (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
        )
     with open(lowerCAmelCase_   ,    'w'   ,    encoding='utf-8'	) as writer:
        for token_index, token in self.ids_to_tokens.items():
           if index != token_index:
              logger.warning(
                  f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                  ' Please check that the vocabulary is not corrupted!'	)
              __lowerCAmelCase				      =      token_index
           writer.write(','.join(lowerCAmelCase_	) + '\n'	)
           index += 1
     with open(lowerCAmelCase_   ,    'w'   ,    encoding='utf-8'	) as writer:
        json.dump(self.emoji   ,    lowerCAmelCase_	)
     return vocab_file, emoji_file





class 							_UpperCAmelCase       (       _UpperCamelCase       ):

  """simple docstring"""


  def __init__(							self					:						Any   ,    lowerCAmelCase_					:						Any   ,    lowerCAmelCase_					:						Tuple   ,    lowerCAmelCase_					:						List[str]	)				->		str:
     __lowerCAmelCase				      =      vocab  # same as swe
     __lowerCAmelCase				      =      ids_to_tokens  # same as bpe
     __lowerCAmelCase				      =      emoji
     __lowerCAmelCase				      =      np.max([len(lowerCAmelCase_	) for w in self.vocab.keys()]	)
     __lowerCAmelCase				      =      re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'	)
     __lowerCAmelCase				      =      re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'	)
     __lowerCAmelCase				      =      re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'	)
     __lowerCAmelCase				      =      re.compile(
         R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'	)
     __lowerCAmelCase				      =      re.compile(
         R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'	)
     __lowerCAmelCase				      =      re.compile(
         R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'	)
     __lowerCAmelCase				      =      '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
     __lowerCAmelCase				      =      '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
     __lowerCAmelCase				      =      str.maketrans({k: '<BLOCK>' for k in keisen + blocks}	)






  def __len__(							self					:						Union[str, Any]	)				->		int:
     return len(self.ids_to_tokens	)






  def 							lowercase						(							self					:						Optional[int]   ,    lowerCAmelCase_					:						Union[str, Any]	)				->		Optional[int]:
     __lowerCAmelCase				      =      self.content_repattera.sub('<URL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<EMAIL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<TEL>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<DATE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<DATE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      self.content_repattera.sub('<PRICE>'   ,    lowerCAmelCase_	)
     __lowerCAmelCase				      =      content.translate(self.content_transa	)
     while "<BLOCK><BLOCK>" in content:
        __lowerCAmelCase				      =      content.replace('<BLOCK><BLOCK>'   ,    '<BLOCK>'	)
     return content






  def 							lowercase						(							self					:						List[str]   ,    lowerCAmelCase_					:						Optional[int]   ,    lowerCAmelCase_					:						List[Any]=False	)				->		Any:
     __lowerCAmelCase				      =      text.replace(' '   ,    '<SP>'	)
     __lowerCAmelCase				      =      text.replace('　'   ,    '<SP>'	)
     __lowerCAmelCase				      =      text.replace('\r\n'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\n'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\r'   ,    '<BR>'	)
     __lowerCAmelCase				      =      text.replace('\t'   ,    '<TAB>'	)
     __lowerCAmelCase				      =      text.replace('—'   ,    'ー'	)
     __lowerCAmelCase				      =      text.replace('−'   ,    'ー'	)
     for k, v in self.emoji["emoji"].items():
        if k in text:
           __lowerCAmelCase				      =      text.replace(lowerCAmelCase_   ,    lowerCAmelCase_	)
     if clean:
        __lowerCAmelCase				      =      self.clean_text(lowerCAmelCase_	)

     def check_simbol(lowerCAmelCase_					:						List[str]	):
        __lowerCAmelCase				      =      x.encode()
        if len(lowerCAmelCase_	) == 1 and len(lowerCAmelCase_	) == 2:
           __lowerCAmelCase				      =      (int(e[0]	) << 8) + int(e[1]	)
           if (
               (c >= 0Xc2_a1 and c <= 0Xc2_bf)
               or (c >= 0Xc7_80 and c <= 0Xc7_83)
               or (c >= 0Xca_b9 and c <= 0Xcb_bf)
               or (c >= 0Xcc_80 and c <= 0Xcd_a2)
           ):
              return True
        return False

     def checkuae(lowerCAmelCase_					:						List[str]	):
        __lowerCAmelCase				      =      x.encode()
        if len(lowerCAmelCase_	) == 1 and len(lowerCAmelCase_	) == 3:
           __lowerCAmelCase				      =      (int(e[0]	) << 1_6) + (int(e[1]	) << 8) + int(e[2]	)
           if c >= 0Xe2_80_80 and c <= 0Xe2_b0_7f:
              return True
        return False

     __lowerCAmelCase				      =      0
     __lowerCAmelCase				      =      []
     while pos < len(lowerCAmelCase_	):
        __lowerCAmelCase				      =      min(len(lowerCAmelCase_	)   ,    pos + self.maxlen + 1	) if text[pos] == '<' else pos + 3
        __lowerCAmelCase				      =      []  # (token_id, token, pos)
        for e in range(lowerCAmelCase_   ,    lowerCAmelCase_   ,    -1	):
           __lowerCAmelCase				      =      text[pos:e]
           if wd in self.vocab:
              if wd[0] == "<" and len(lowerCAmelCase_	) > 2:
                 __lowerCAmelCase				      =      [(self.vocab[wd], wd, e)]
                 break
              else:
                 candidates.append((self.vocab[wd], wd, e)	)
        if len(lowerCAmelCase_	) > 0:
           # the smallest token_id is adopted
           __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase				      =      sorted(lowerCAmelCase_   ,    key=lambda lowerCAmelCase_	: x[0]	)[0]
           result.append(lowerCAmelCase_	)
           __lowerCAmelCase				      =      e
        else:
           __lowerCAmelCase				      =      pos + 1
           __lowerCAmelCase				      =      text[pos:end]
           if check_simbol(lowerCAmelCase_	):
              result.append('<KIGOU>'	)
           elif checkuae(lowerCAmelCase_	):
              result.append('<U2000U2BFF>'	)
           else:
              for i in wd.encode('utf-8'	):
                 result.append('<|byte%d|>' % i	)
           __lowerCAmelCase				      =      end
     return result






  def 							lowercase						(							self					:						Dict   ,    lowerCAmelCase_					:						Any   ,    lowerCAmelCase_					:						List[Any]="\n"	)				->		Tuple:
     __lowerCAmelCase				      =      []
     __lowerCAmelCase				      =      []
     __lowerCAmelCase				      =      self.ids_to_tokens[index][0]
     if word[:6] == "<|byte" and word[-2:] == "|>":
        byte_tokens.append(int(word[6:-2]	)	)
     else:
        if len(lowerCAmelCase_	) > 0:
           words.append(bytearray(lowerCAmelCase_	).decode('utf-8'   ,    errors='replace'	)	)
           __lowerCAmelCase				      =      []
        if word[:7] == "<|emoji" and word[-2:] == "|>":
           words.append(self.emoji['emoji_inv'][word]	)
        elif word == "<SP>":
           words.append(' '	)
        elif word == "<BR>":
           words.append(lowerCAmelCase_	)
        elif word == "<TAB>":
           words.append('\t'	)
        elif word == "<BLOCK>":
           words.append('▀'	)
        elif word == "<KIGOU>":
           words.append('ǀ'	)
        elif word == "<U2000U2BFF>":
           words.append('‖'	)
        else:
           words.append(lowerCAmelCase_	)
     if len(lowerCAmelCase_	) > 0:
        words.append(bytearray(lowerCAmelCase_	).decode('utf-8'   ,    errors='replace'	)	)
     __lowerCAmelCase				      =      ''.join(lowerCAmelCase_	)
     return text








"""simple docstring"""




import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class 			__UpperCamelCase							(		unittest.TestCase		):
							lowerCamelCase  :   Union[str, Any]    =MODEL_FOR_MASKED_LM_MAPPING
							lowerCamelCase  :   List[str]    =TF_MODEL_FOR_MASKED_LM_MAPPING






							def        __a  (  self )      ->					Tuple:
								super().tearDown()
								# clean-up as much as possible GPU memory occupied by PyTorch
								gc.collect()
								if is_torch_available():
									import torch

									torch.cuda.empty_cache()






							@require_tf
							def        __a  (  self )      ->					List[Any]:
								a :						Dict      =							pipeline(task="fill-mask" ,   model="sshleifer/tiny-distilroberta-base" ,   top_k=2 ,   framework="tf" )
								a :						Optional[Any]      =							unmasker("My name is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {"sequence": "My name is grouped", "score": 2.1E-05, "token": 3_8015, "token_str": " grouped"},
								        {"sequence": "My name is accuser", "score": 2.1E-05, "token": 2_5506, "token_str": " accuser"},
								    ] ,   )

								a :						str      =							unmasker("The largest city in France is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {
								            "sequence": "The largest city in France is grouped",
								            "score": 2.1E-05,
								            "token": 3_8015,
								            "token_str": " grouped",
								        },
								        {
								            "sequence": "The largest city in France is accuser",
								            "score": 2.1E-05,
								            "token": 2_5506,
								            "token_str": " accuser",
								        },
								    ] ,   )

								a :						Any      =							unmasker("My name is <mask>" ,   targets=[" Patrick", " Clara", " Teven"] ,   top_k=3 )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {"sequence": "My name is Clara", "score": 2E-05, "token": 1_3606, "token_str": " Clara"},
								        {"sequence": "My name is Patrick", "score": 2E-05, "token": 3499, "token_str": " Patrick"},
								        {"sequence": "My name is Te", "score": 1.9E-05, "token": 2941, "token_str": " Te"},
								    ] ,   )






							@require_torch
							def        __a  (  self )      ->					Tuple:
								a :						Optional[int]      =							pipeline(task="fill-mask" ,   model="sshleifer/tiny-distilroberta-base" ,   top_k=2 ,   framework="pt" )

								a :						int      =							unmasker("My name is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {"sequence": "My name is Maul", "score": 2.2E-05, "token": 3_5676, "token_str": " Maul"},
								        {"sequence": "My name isELS", "score": 2.2E-05, "token": 1_6416, "token_str": "ELS"},
								    ] ,   )

								a :						Optional[Any]      =							unmasker("The largest city in France is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {
								            "sequence": "The largest city in France is Maul",
								            "score": 2.2E-05,
								            "token": 3_5676,
								            "token_str": " Maul",
								        },
								        {"sequence": "The largest city in France isELS", "score": 2.2E-05, "token": 1_6416, "token_str": "ELS"},
								    ] ,   )

								a :						List[str]      =							unmasker("My name is <mask>" ,   targets=[" Patrick", " Clara", " Teven"] ,   top_k=3 )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        {"sequence": "My name is Patrick", "score": 2.1E-05, "token": 3499, "token_str": " Patrick"},
								        {"sequence": "My name is Te", "score": 2E-05, "token": 2941, "token_str": " Te"},
								        {"sequence": "My name is Clara", "score": 2E-05, "token": 1_3606, "token_str": " Clara"},
								    ] ,   )

								a :						Optional[int]      =							unmasker("My name is <mask> <mask>" ,   top_k=2 )

								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ,   decimals=6 ) ,   [
								        [
								            {
								                "score": 2.2E-05,
								                "token": 3_5676,
								                "token_str": " Maul",
								                "sequence": "<s>My name is Maul<mask></s>",
								            },
								            {"score": 2.2E-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name isELS<mask></s>"},
								        ],
								        [
								            {
								                "score": 2.2E-05,
								                "token": 3_5676,
								                "token_str": " Maul",
								                "sequence": "<s>My name is<mask> Maul</s>",
								            },
								            {"score": 2.2E-05, "token": 1_6416, "token_str": "ELS", "sequence": "<s>My name is<mask>ELS</s>"},
								        ],
								    ] ,   )






							@require_torch_gpu
							def        __a  (  self )      ->					Optional[Any]:
								a :						List[Any]      =							pipeline("fill-mask" ,   model="hf-internal-testing/tiny-random-distilbert" ,   device=0 ,   framework="pt" )

								# convert model to fp16
								pipe.model.half()

								a :						Optional[Any]      =							pipe("Paris is the [MASK] of France." )
								# We actually don't care about the result, we just want to make sure
								# it works, meaning the float16 tensor got casted back to float32
								# for postprocessing.
								self.assertIsInstance(lowerCAmelCase__ ,   lowerCAmelCase__ )






							@slow
							@require_torch
							def        __a  (  self )      ->					Any:
								a :						List[Any]      =							pipeline(task="fill-mask" ,   model="distilroberta-base" ,   top_k=2 ,   framework="pt" )
								self.run_large_test(lowerCAmelCase__ )






							@slow
							@require_tf
							def        __a  (  self )      ->					Any:
								a :						int      =							pipeline(task="fill-mask" ,   model="distilroberta-base" ,   top_k=2 ,   framework="tf" )
								self.run_large_test(lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					List[Any]:
								a :						List[str]      =							unmasker("My name is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ) ,   [
								        {"sequence": "My name is John", "score": 0.008, "token": 610, "token_str": " John"},
								        {"sequence": "My name is Chris", "score": 0.007, "token": 1573, "token_str": " Chris"},
								    ] ,   )
								a :						Union[str, Any]      =							unmasker("The largest city in France is <mask>" )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ) ,   [
								        {
								            "sequence": "The largest city in France is Paris",
								            "score": 0.251,
								            "token": 2201,
								            "token_str": " Paris",
								        },
								        {
								            "sequence": "The largest city in France is Lyon",
								            "score": 0.214,
								            "token": 1_2790,
								            "token_str": " Lyon",
								        },
								    ] ,   )

								a :						Tuple      =							unmasker("My name is <mask>" ,   targets=[" Patrick", " Clara", " Teven"] ,   top_k=3 )
								self.assertEqual(
								    nested_simplify(lowerCAmelCase__ ) ,   [
								        {"sequence": "My name is Patrick", "score": 0.005, "token": 3499, "token_str": " Patrick"},
								        {"sequence": "My name is Clara", "score": 0.000, "token": 1_3606, "token_str": " Clara"},
								        {"sequence": "My name is Te", "score": 0.000, "token": 2941, "token_str": " Te"},
								    ] ,   )






							@require_torch
							def        __a  (  self )      ->					str:
								a :						Any      =							pipeline(task="fill-mask" ,   model="sshleifer/tiny-distilroberta-base" ,   framework="pt" )
								a :						List[str]      =							None
								a :						Dict      =							None
								self.run_pipeline_test(lowerCAmelCase__ ,   [] )






							@require_tf
							def        __a  (  self )      ->					str:
								a :						Optional[Any]      =							pipeline(task="fill-mask" ,   model="sshleifer/tiny-distilroberta-base" ,   framework="tf" )
								a :						Any      =							None
								a :						Optional[int]      =							None
								self.run_pipeline_test(lowerCAmelCase__ ,   [] )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Tuple:
								if tokenizer is None or tokenizer.mask_token_id is None:
									self.skipTest("The provided tokenizer has no mask token, (probably reformer or wav2vec2)" )

								a :						str      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )
								a :						Optional[Any]      =							[
								    f"""This is another {tokenizer.mask_token} test""",
								]
								return fill_masker, examples






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					str:
								a :						Any      =							fill_masker.tokenizer
								a :						Optional[int]      =							fill_masker.model

								a :						List[str]      =							fill_masker(
								    f"""This is a {tokenizer.mask_token}""" ,   )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )

								a :						int      =							fill_masker([f"""This is a {tokenizer.mask_token}"""] )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )

								a :						Tuple      =							fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""] )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        [
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        ],
								        [
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        ],
								    ] ,   )

								with self.assertRaises(lowerCAmelCase__ ):
									fill_masker([None] )
								# No mask_token is not supported
								with self.assertRaises(lowerCAmelCase__ ):
									fill_masker("This is" )

								self.run_test_top_k(lowerCAmelCase__ ,   lowerCAmelCase__ )
								self.run_test_targets(lowerCAmelCase__ ,   lowerCAmelCase__ )
								self.run_test_top_k_targets(lowerCAmelCase__ ,   lowerCAmelCase__ )
								self.fill_mask_with_duplicate_targets_and_top_k(lowerCAmelCase__ ,   lowerCAmelCase__ )
								self.fill_mask_with_multiple_masks(lowerCAmelCase__ ,   lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Dict:
								a :						Union[str, Any]      =							tokenizer.get_vocab()
								a :						List[Any]      =							sorted(vocab.keys() )[:2]
								# Pipeline argument
								a :						str      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ ,   targets=lowerCAmelCase__ )
								a :						Dict      =							fill_masker(f"""This is a {tokenizer.mask_token}""" )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )
								a :						List[str]      =							{vocab[el] for el in targets}
								self.assertEqual({el["token"] for el in outputs} ,   lowerCAmelCase__ )
								a :						str      =							[tokenizer.decode([x] ) for x in target_ids]
								self.assertEqual({el["token_str"] for el in outputs} ,   set(lowerCAmelCase__ ) )

								# Call argument
								a :						Optional[Any]      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )
								a :						int      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets=lowerCAmelCase__ )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )
								a :						Any      =							{vocab[el] for el in targets}
								self.assertEqual({el["token"] for el in outputs} ,   lowerCAmelCase__ )
								a :						Optional[Any]      =							[tokenizer.decode([x] ) for x in target_ids]
								self.assertEqual({el["token_str"] for el in outputs} ,   set(lowerCAmelCase__ ) )

								# Score equivalence
								a :						Tuple      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets=lowerCAmelCase__ )
								a :						List[str]      =							[top_mask["token_str"] for top_mask in outputs]
								a :						Dict      =							[top_mask["score"] for top_mask in outputs]

								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(lowerCAmelCase__ ) == set(lowerCAmelCase__ ):
									a :						Union[str, Any]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets=lowerCAmelCase__ )
									a :						Tuple      =							[top_mask["score"] for top_mask in unmasked_targets]
									self.assertEqual(nested_simplify(lowerCAmelCase__ ) ,   nested_simplify(lowerCAmelCase__ ) )

								# Raises with invalid
								with self.assertRaises(lowerCAmelCase__ ):
									a :						Tuple      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets=[] )
								# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
								if "" not in tokenizer.get_vocab():
									with self.assertRaises(lowerCAmelCase__ ):
										a :						List[Any]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets=[""] )
									with self.assertRaises(lowerCAmelCase__ ):
										a :						Optional[int]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   targets="" )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Optional[Any]:
								a :						int      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ ,   top_k=2 )
								a :						List[Any]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )

								a :						Optional[Any]      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )
								a :						Optional[int]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   top_k=2 )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								    ] ,   )
								self.assertEqual(nested_simplify(lowerCAmelCase__ ) ,   nested_simplify(lowerCAmelCase__ ) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					str:
								a :						Tuple      =							tokenizer.get_vocab()
								a :						List[str]      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )

								# top_k=2, ntargets=3
								a :						Tuple      =							sorted(vocab.keys() )[:3]
								a :						Tuple      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   top_k=2 ,   targets=lowerCAmelCase__ )

								# If we use the most probably targets, and filter differently, we should still
								# have the same results
								a :						str      =							[el["token_str"] for el in sorted(lowerCAmelCase__ ,   key=lambda lowerCAmelCase__ : x["score"] ,   reverse=lowerCAmelCase__ )]
								# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
								if set(lowerCAmelCase__ ).issubset(lowerCAmelCase__ ):
									a :						Optional[int]      =							fill_masker(f"""This is a {tokenizer.mask_token}""" ,   top_k=3 ,   targets=lowerCAmelCase__ )
									# They should yield exactly the same result
									self.assertEqual(nested_simplify(lowerCAmelCase__ ) ,   nested_simplify(lowerCAmelCase__ ) )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Any:
								a :						Tuple      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )
								a :						Union[str, Any]      =							tokenizer.get_vocab()
								# String duplicates + id duplicates
								a :						Optional[Any]      =							sorted(vocab.keys() )[:3]
								a :						int      =							[targets[0], targets[1], targets[0], targets[2], targets[1]]
								a :						Tuple      =							fill_masker(f"""My name is {tokenizer.mask_token}""" ,   targets=lowerCAmelCase__ ,   top_k=10 )

								# The target list contains duplicates, so we can't output more
								# than them
								self.assertEqual(len(lowerCAmelCase__ ) ,   3 )






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ )      ->					Optional[int]:
								a :						str      =							FillMaskPipeline(model=lowerCAmelCase__ ,   tokenizer=lowerCAmelCase__ )

								a :						Dict      =							fill_masker(
								    f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}""" ,   top_k=2 )
								self.assertEqual(
								    lowerCAmelCase__ ,   [
								        [
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        ],
								        [
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        ],
								        [
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								            {"sequence": ANY(lowerCAmelCase__ ), "score": ANY(lowerCAmelCase__ ), "token": ANY(lowerCAmelCase__ ), "token_str": ANY(lowerCAmelCase__ )},
								        ],
								    ] ,   )


