'''simple docstring'''




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


__lowercase          =  logging.get_logger(__name__)

__lowercase          =  {'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

__lowercase          =  {
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

__lowercase          =  {
    '''abeja/gpt-neox-japanese-2.7b''': 2_0_4_8,
}






def 				snake_case__							(			_A:					Optional[Any]      ,   _A:					Optional[Any] )    ->	Optional[int]:

		'''simple docstring'''


		with open(_A      ,   """r"""      ,   encoding="""utf-8""" ) as f:
				lowerCAmelCase   					=  json.loads(f.read() )

		lowerCAmelCase   					=  collections.OrderedDict()
		lowerCAmelCase   					=  collections.OrderedDict()
		lowerCAmelCase   					=  collections.OrderedDict()
		with open(_A      ,   """r"""      ,   encoding="""utf-8""" ) as f:
				lowerCAmelCase   					=  f.readlines()
		lowerCAmelCase   					=  [[t.rstrip("""\n""" )] if (t == """,""" or """,""" not in t) else t.rstrip("""\n""" ).split(""",""" ) for t in token]
		for idx, b in enumerate(_A ):
				lowerCAmelCase   					=  b
				lowerCAmelCase   					=  idx
				for wd in b:
						lowerCAmelCase   					=  idx

		return vocab, raw_vocab, ids_to_tokens, emoji






class 				a__(					lowerCAmelCase__	):


			'''simple docstring'''






			UpperCAmelCase_			:     List[str]      =       VOCAB_FILES_NAMES
			UpperCAmelCase_			:     int      =       PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_			:     int      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_			:     str      =       ['''input_ids''', '''attention_mask''']



			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase="<|endoftext|>"       ,    __lowerCAmelCase="<|endoftext|>"       ,    __lowerCAmelCase="<|startoftext|>"       ,    __lowerCAmelCase="<|endoftext|>"       ,    __lowerCAmelCase=False       ,    **__lowerCAmelCase       ,    ):




					"""simple docstring"""


					super().__init__(
					    unk_token=__lowerCAmelCase       ,    pad_token=__lowerCAmelCase       ,    bos_token=__lowerCAmelCase       ,    eos_token=__lowerCAmelCase       ,    do_clean_text=__lowerCAmelCase       ,    **__lowerCAmelCase       ,    )
					if not os.path.isfile(__lowerCAmelCase):
							raise ValueError(
							    f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
							    """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`""")
					if not os.path.isfile(__lowerCAmelCase):
							raise ValueError(
							    f"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
							    """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`""")
					lowerCAmelCase   					=  do_clean_text
					lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  load_vocab_and_emoji(__lowerCAmelCase       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  SubWordJapaneseTokenizer(
					    vocab=self.vocab       ,    ids_to_tokens=self.ids_to_tokens       ,    emoji=self.emoji)



			@property
			def 							a_       (   self):




					"""simple docstring"""


					return len(self.raw_vocab)



			def 							a_       (   self):




					"""simple docstring"""


					return dict(self.raw_vocab       ,    **self.added_tokens_encoder)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					return self.subword_tokenizer.tokenize(__lowerCAmelCase       ,    clean=self.do_clean_text)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					return self.vocab.get(__lowerCAmelCase       ,    self.vocab.get(self.unk_token))



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					return self.subword_tokenizer.convert_id_to_token(__lowerCAmelCase)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  """""".join(__lowerCAmelCase).strip()
					return out_string



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  []
					for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(__lowerCAmelCase       ,    add_special_tokens=__lowerCAmelCase) + [self.eos_token_id])

					if len(__lowerCAmelCase) > self.model_max_length:
							lowerCAmelCase   					=  input_ids[-self.model_max_length :]
					return input_ids







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase   					=  0
					if os.path.isdir(__lowerCAmelCase):
							lowerCAmelCase   					=  os.path.join(
							    __lowerCAmelCase       ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
							lowerCAmelCase   					=  os.path.join(
							    __lowerCAmelCase       ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""])
					else:
							lowerCAmelCase   					=  (
							    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
							)
							lowerCAmelCase   					=  (
							    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
							)
					with open(__lowerCAmelCase       ,    """w"""       ,    encoding="""utf-8""") as writer:
							for token_index, token in self.ids_to_tokens.items():
									if index != token_index:
											logger.warning(
											    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
											    """ Please check that the vocabulary is not corrupted!""")
											lowerCAmelCase   					=  token_index
									writer.write(""",""".join(__lowerCAmelCase) + """\n""")
									index += 1
					with open(__lowerCAmelCase       ,    """w"""       ,    encoding="""utf-8""") as writer:
							json.dump(self.emoji       ,    __lowerCAmelCase)
					return vocab_file, emoji_file







class 				a__(					lowerCAmelCase__	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  vocab  # same as swe
					lowerCAmelCase   					=  ids_to_tokens  # same as bpe
					lowerCAmelCase   					=  emoji
					lowerCAmelCase   					=  np.max([len(__lowerCAmelCase) for w in self.vocab.keys()])
					lowerCAmelCase   					=  re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)""")
					lowerCAmelCase   					=  re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*""")
					lowerCAmelCase   					=  re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}""")
					lowerCAmelCase   					=  re.compile(
					    r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*""")
					lowerCAmelCase   					=  re.compile(
					    r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*""")
					lowerCAmelCase   					=  re.compile(
					    r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*""")
					lowerCAmelCase   					=  """─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
					lowerCAmelCase   					=  """▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
					lowerCAmelCase   					=  str.maketrans({k: """<BLOCK>""" for k in keisen + blocks})



			def __len__(   self):




					"""simple docstring"""


					return len(self.ids_to_tokens)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  self.content_repattera.sub("""<URL>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  self.content_repattera.sub("""<EMAIL>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  self.content_repattera.sub("""<TEL>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  self.content_repattera.sub("""<DATE>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  self.content_repattera.sub("""<DATE>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  self.content_repattera.sub("""<PRICE>"""       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  content.translate(self.content_transa)
					while "<BLOCK><BLOCK>" in content:
							lowerCAmelCase   					=  content.replace("""<BLOCK><BLOCK>"""       ,    """<BLOCK>""")
					return content



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase=False):




					"""simple docstring"""


					lowerCAmelCase   					=  text.replace(""" """       ,    """<SP>""")
					lowerCAmelCase   					=  text.replace("""　"""       ,    """<SP>""")
					lowerCAmelCase   					=  text.replace("""\r\n"""       ,    """<BR>""")
					lowerCAmelCase   					=  text.replace("""\n"""       ,    """<BR>""")
					lowerCAmelCase   					=  text.replace("""\r"""       ,    """<BR>""")
					lowerCAmelCase   					=  text.replace("""\t"""       ,    """<TAB>""")
					lowerCAmelCase   					=  text.replace("""—"""       ,    """ー""")
					lowerCAmelCase   					=  text.replace("""−"""       ,    """ー""")
					for k, v in self.emoji["emoji"].items():
							if k in text:
									lowerCAmelCase   					=  text.replace(__lowerCAmelCase       ,    __lowerCAmelCase)
					if clean:
							lowerCAmelCase   					=  self.clean_text(__lowerCAmelCase)

					def check_simbol(__lowerCAmelCase):
							lowerCAmelCase   					=  x.encode()
							if len(__lowerCAmelCase) == 1 and len(__lowerCAmelCase) == 2:
									lowerCAmelCase   					=  (int(e[0]) << 8) + int(e[1])
									if (
									    (c >= 0Xc2a1 and c <= 0Xc2bf)
									    or (c >= 0Xc780 and c <= 0Xc783)
									    or (c >= 0Xcab9 and c <= 0Xcbbf)
									    or (c >= 0Xcc80 and c <= 0Xcda2)
									):
											return True
							return False

					def checkuae(__lowerCAmelCase):
							lowerCAmelCase   					=  x.encode()
							if len(__lowerCAmelCase) == 1 and len(__lowerCAmelCase) == 3:
									lowerCAmelCase   					=  (int(e[0]) << 16) + (int(e[1]) << 8) + int(e[2])
									if c >= 0Xe2_8080 and c <= 0Xe2_b07f:
											return True
							return False

					lowerCAmelCase   					=  0
					lowerCAmelCase   					=  []
					while pos < len(__lowerCAmelCase):
							lowerCAmelCase   					=  min(len(__lowerCAmelCase)       ,    pos + self.maxlen + 1) if text[pos] == """<""" else pos + 3
							lowerCAmelCase   					=  []  # (token_id, token, pos)
							for e in range(__lowerCAmelCase       ,    __lowerCAmelCase       ,    -1):
									lowerCAmelCase   					=  text[pos:e]
									if wd in self.vocab:
											if wd[0] == "<" and len(__lowerCAmelCase) > 2:
													lowerCAmelCase   					=  [(self.vocab[wd], wd, e)]
													break
											else:
													candidates.append((self.vocab[wd], wd, e))
							if len(__lowerCAmelCase) > 0:
									# the smallest token_id is adopted
									lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  sorted(__lowerCAmelCase       ,    key=lambda __lowerCAmelCase: x[0])[0]
									result.append(__lowerCAmelCase)
									lowerCAmelCase   					=  e
							else:
									lowerCAmelCase   					=  pos + 1
									lowerCAmelCase   					=  text[pos:end]
									if check_simbol(__lowerCAmelCase):
											result.append("""<KIGOU>""")
									elif checkuae(__lowerCAmelCase):
											result.append("""<U2000U2BFF>""")
									else:
											for i in wd.encode("""utf-8"""):
													result.append("""<|byte%d|>""" % i)
									lowerCAmelCase   					=  end
					return result







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase="\n"):




					"""simple docstring"""


					lowerCAmelCase   					=  []
					lowerCAmelCase   					=  []
					lowerCAmelCase   					=  self.ids_to_tokens[index][0]
					if word[:6] == "<|byte" and word[-2:] == "|>":
							byte_tokens.append(int(word[6:-2]))
					else:
							if len(__lowerCAmelCase) > 0:
									words.append(bytearray(__lowerCAmelCase).decode("""utf-8"""       ,    errors="""replace"""))
									lowerCAmelCase   					=  []
							if word[:7] == "<|emoji" and word[-2:] == "|>":
									words.append(self.emoji["""emoji_inv"""][word])
							elif word == "<SP>":
									words.append(""" """)
							elif word == "<BR>":
									words.append(__lowerCAmelCase)
							elif word == "<TAB>":
									words.append("""\t""")
							elif word == "<BLOCK>":
									words.append("""▀""")
							elif word == "<KIGOU>":
									words.append("""ǀ""")
							elif word == "<U2000U2BFF>":
									words.append("""‖""")
							else:
									words.append(__lowerCAmelCase)
					if len(__lowerCAmelCase) > 0:
							words.append(bytearray(__lowerCAmelCase).decode("""utf-8"""       ,    errors="""replace"""))
					lowerCAmelCase   					=  """""".join(__lowerCAmelCase)
					return text





'''simple docstring'''




import os
import string
import sys


__lowercase          =  1 << 8

__lowercase          =  {
    '''tab''': ord('''\t'''),
    '''newline''': ord('''\r'''),
    '''esc''': 2_7,
    '''up''': 6_5 + ARROW_KEY_FLAG,
    '''down''': 6_6 + ARROW_KEY_FLAG,
    '''right''': 6_7 + ARROW_KEY_FLAG,
    '''left''': 6_8 + ARROW_KEY_FLAG,
    '''mod_int''': 9_1,
    '''undefined''': sys.maxsize,
    '''interrupt''': 3,
    '''insert''': 5_0,
    '''delete''': 5_1,
    '''pg_up''': 5_3,
    '''pg_down''': 5_4,
}

__lowercase          =  KEYMAP['''up''']
__lowercase          =  KEYMAP['''left''']

if sys.platform == "win32":
						__lowercase          =  []
						__lowercase          =  {
						    B'''\xe0H''': KEYMAP['''up'''] - ARROW_KEY_FLAG,
						    B'''\x00H''': KEYMAP['''up'''] - ARROW_KEY_FLAG,
						    B'''\xe0P''': KEYMAP['''down'''] - ARROW_KEY_FLAG,
						    B'''\x00P''': KEYMAP['''down'''] - ARROW_KEY_FLAG,
						    B'''\xe0M''': KEYMAP['''right'''] - ARROW_KEY_FLAG,
						    B'''\x00M''': KEYMAP['''right'''] - ARROW_KEY_FLAG,
						    B'''\xe0K''': KEYMAP['''left'''] - ARROW_KEY_FLAG,
						    B'''\x00K''': KEYMAP['''left'''] - ARROW_KEY_FLAG,
						}

for i in range(1_0):
						__lowercase          =  ord(str(i))






def 				snake_case__							(			)    ->	List[Any]:

		'''simple docstring'''


		if os.name == "nt":
				import msvcrt

				lowerCAmelCase   					=  """mbcs"""
				# Flush the keyboard buffer
				while msvcrt.kbhit():
						msvcrt.getch()
				if len(_A ) == 0:
						# Read the keystroke
						lowerCAmelCase   					=  msvcrt.getch()

						# If it is a prefix char, get second part
						if ch in (b"\x00", b"\xe0"):
								lowerCAmelCase   					=  ch + msvcrt.getch()
								# Translate actual Win chars to bullet char types
								try:
										lowerCAmelCase   					=  chr(WIN_KEYMAP[cha] )
										WIN_CH_BUFFER.append(chr(KEYMAP["""mod_int"""] ) )
										WIN_CH_BUFFER.append(_A )
										if ord(_A ) in (
										    KEYMAP["insert"] - 1 << 9,
										    KEYMAP["delete"] - 1 << 9,
										    KEYMAP["pg_up"] - 1 << 9,
										    KEYMAP["pg_down"] - 1 << 9,
										):
												WIN_CH_BUFFER.append(chr(126 ) )
										lowerCAmelCase   					=  chr(KEYMAP["""esc"""] )
								except KeyError:
										lowerCAmelCase   					=  cha[1]
						else:
								lowerCAmelCase   					=  ch.decode(_A )
				else:
						lowerCAmelCase   					=  WIN_CH_BUFFER.pop(0 )
		elif os.name == "posix":
				import termios
				import tty

				lowerCAmelCase   					=  sys.stdin.fileno()
				lowerCAmelCase   					=  termios.tcgetattr(_A )
				try:
						tty.setraw(_A )
						lowerCAmelCase   					=  sys.stdin.read(1 )
				finally:
						termios.tcsetattr(_A      ,   termios.TCSADRAIN      ,   _A )
		return ch






def 				snake_case__							(			)    ->	Tuple:

		'''simple docstring'''


		lowerCAmelCase   					=  get_raw_chars()
		if ord(_A ) in [KEYMAP["interrupt"], KEYMAP["newline"]]:
				return char

		elif ord(_A ) == KEYMAP["esc"]:
				lowerCAmelCase   					=  get_raw_chars()
				if ord(_A ) == KEYMAP["mod_int"]:
						lowerCAmelCase   					=  get_raw_chars()
						if ord(_A ) >= KEYMAP["arrow_begin"] - ARROW_KEY_FLAG and ord(_A ) <= KEYMAP["arrow_end"] - ARROW_KEY_FLAG:
								return chr(ord(_A ) + ARROW_KEY_FLAG )
						else:
								return KEYMAP["undefined"]
				else:
						return get_raw_chars()

		else:
				if char in string.printable:
						return char
				else:
						return KEYMAP["undefined"]




