





"""simple docstring"""






import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


__UpperCAmelCase						=			logging.get_logger(__name__)

__UpperCAmelCase						=			{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

__UpperCAmelCase						=			{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

__UpperCAmelCase						=			{
    '''abeja/gpt-neox-japanese-2.7b''': 2048,
}






def      lowercase__ (					lowerCAmelCase__   :			Optional[int]							,			lowerCAmelCase__   :			Optional[int]			)    ->			Dict:



			'''simple docstring'''
			with open(lowerCAmelCase__							,			"r"							,			encoding="utf-8"			) as f:
						a__  :	str		 =			json.loads(f.read()			)

			a__  :	Union[str, Any]		 =			collections.OrderedDict()
			a__  :	Tuple		 =			collections.OrderedDict()
			a__  :	Optional[Any]		 =			collections.OrderedDict()
			with open(lowerCAmelCase__							,			"r"							,			encoding="utf-8"			) as f:
						a__  :	Union[str, Any]		 =			f.readlines()
			a__  :	Optional[Any]		 =			[[t.rstrip("\n"			)] if (t == "," or "," not in t) else t.rstrip("\n"			).split(","			) for t in token]
			for idx, b in enumerate(lowerCAmelCase__			):
						a__  :	List[Any]		 =			b
						a__  :	str		 =			idx
						for wd in b:
									a__  :	int		 =			idx

			return vocab, raw_vocab, ids_to_tokens, emoji



class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				__lowerCamelCase				:	Tuple				     =				VOCAB_FILES_NAMES
				__lowerCamelCase				:	List[str]				     =				PRETRAINED_VOCAB_FILES_MAP
				__lowerCamelCase				:	Optional[Any]				     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__lowerCamelCase				:	str				     =				["input_ids", "attention_mask"]
				def __init__(		self    :			Union[str, Any]	, a_    :			Tuple	, a_    :			Tuple	, a_    :			List[Any]="<|endoftext|>"	, a_    :			str="<|endoftext|>"	, a_    :			Tuple="<|startoftext|>"	, a_    :			List[Any]="<|endoftext|>"	, a_    :			Dict=False	, **a_    :			Any	, )	->	int:



							'''simple docstring'''



							super().__init__(
							    unk_token=a_	, pad_token=a_	, bos_token=a_	, eos_token=a_	, do_clean_text=a_	, **a_	, )
							if not os.path.isfile(a_    ):
										raise ValueError(
										    F"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
										    " model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"    )
							if not os.path.isfile(a_    ):
										raise ValueError(
										    F"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
										    " pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"    )
							a__  :	Optional[int]		 =			do_clean_text
							a__ ,	a__ ,	a__ ,	a__  :	Any		 =			load_vocab_and_emoji(a_	, a_    )
							a__  :	Optional[Any]		 =			SubWordJapaneseTokenizer(
							    vocab=self.vocab	, ids_to_tokens=self.ids_to_tokens	, emoji=self.emoji    )
				@property
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Optional[Any]:



							'''simple docstring'''



							return len(self.raw_vocab    )
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Optional[int]:



							'''simple docstring'''



							return dict(self.raw_vocab	, **self.added_tokens_encoder    )
				def       UpperCAmelCase      (		self    :			Tuple	, a_    :			Any    )	->	List[str]:



							'''simple docstring'''



							return self.subword_tokenizer.tokenize(a_	, clean=self.do_clean_text    )
				def       UpperCAmelCase      (		self    :			str	, a_    :			List[Any]    )	->	int:



							'''simple docstring'''



							return self.vocab.get(a_	, self.vocab.get(self.unk_token    )    )
				def       UpperCAmelCase      (		self    :			int	, a_    :			Any    )	->	List[Any]:



							'''simple docstring'''



							return self.subword_tokenizer.convert_id_to_token(a_    )
				def       UpperCAmelCase      (		self    :			List[Any]	, a_    :			Optional[int]    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	Dict		 =			"".join(a_    ).strip()
							return out_string
				def       UpperCAmelCase      (		self    :			Union[str, Any]	, a_    :			"Conversation"    )	->	List[int]:



							'''simple docstring'''



							a__  :	List[str]		 =			[]
							for is_user, text in conversation.iter_texts():
										input_ids.extend(self.encode(a_	, add_special_tokens=a_    ) + [self.eos_token_id]    )

							if len(a_    ) > self.model_max_length:
										a__  :	str		 =			input_ids[-self.model_max_length :]
							return input_ids






				def       UpperCAmelCase      (		self    :			Any	, a_    :			str	, a_    :			Optional[str] = None    )	->	Tuple[str]:



							'''simple docstring'''



							a__  :	Dict		 =			0
							if os.path.isdir(a_    ):
										a__  :	Optional[Any]		 =			os.path.join(
										    a_	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
										a__  :	List[str]		 =			os.path.join(
										    a_	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["emoji_file"]    )
							else:
										a__  :	int		 =			(
										    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["vocab_file"]
										)
										a__  :	Optional[int]		 =			(
										    (filename_prefix + "-" if filename_prefix else "") + save_directory + VOCAB_FILES_NAMES["emoji_file"]
										)
							with open(a_	, "w"	, encoding="utf-8"    ) as writer:
										for token_index, token in self.ids_to_tokens.items():
													if index != token_index:
																logger.warning(
																    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																    " Please check that the vocabulary is not corrupted!"    )
																a__  :	Tuple		 =			token_index
													writer.write(",".join(a_    ) + "\n"    )
													index += 1
							with open(a_	, "w"	, encoding="utf-8"    ) as writer:
										json.dump(self.emoji	, a_    )
							return vocab_file, emoji_file



class 					__UpperCAmelCase							(						_UpperCamelCase  ):
				def __init__(		self    :			Optional[int]	, a_    :			Dict	, a_    :			Optional[Any]	, a_    :			int    )	->	Tuple:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			vocab  # same as swe
							a__  :	List[Any]		 =			ids_to_tokens  # same as bpe
							a__  :	Dict		 =			emoji
							a__  :	int		 =			np.max([len(a_    ) for w in self.vocab.keys()]    )
							a__  :	int		 =			re.compile(R"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"    )
							a__  :	Tuple		 =			re.compile(R"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"    )
							a__  :	Optional[int]		 =			re.compile(R"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"    )
							a__  :	Any		 =			re.compile(
							    R"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"    )
							a__  :	List[str]		 =			re.compile(
							    R"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"    )
							a__  :	Tuple		 =			re.compile(
							    R"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"    )
							a__  :	List[str]		 =			"─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
							a__  :	Optional[Any]		 =			"▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
							a__  :	Union[str, Any]		 =			str.maketrans({k: "<BLOCK>" for k in keisen + blocks}    )
				def __len__(		self    :			Dict    )	->	Optional[int]:



							'''simple docstring'''



							return len(self.ids_to_tokens    )
				def       UpperCAmelCase      (		self    :			Optional[int]	, a_    :			Any    )	->	Any:



							'''simple docstring'''



							a__  :	int		 =			self.content_repattera.sub("<URL>"	, a_    )
							a__  :	Optional[Any]		 =			self.content_repattera.sub("<EMAIL>"	, a_    )
							a__  :	Any		 =			self.content_repattera.sub("<TEL>"	, a_    )
							a__  :	Union[str, Any]		 =			self.content_repattera.sub("<DATE>"	, a_    )
							a__  :	Any		 =			self.content_repattera.sub("<DATE>"	, a_    )
							a__  :	Any		 =			self.content_repattera.sub("<PRICE>"	, a_    )
							a__  :	List[Any]		 =			content.translate(self.content_transa    )
							while "<BLOCK><BLOCK>" in content:
										a__  :	Tuple		 =			content.replace("<BLOCK><BLOCK>"	, "<BLOCK>"    )
							return content
				def       UpperCAmelCase      (		self    :			int	, a_    :			int	, a_    :			List[str]=False    )	->	Tuple:



							'''simple docstring'''



							a__  :	Dict		 =			text.replace(" "	, "<SP>"    )
							a__  :	Optional[Any]		 =			text.replace("　"	, "<SP>"    )
							a__  :	List[str]		 =			text.replace("\r\n"	, "<BR>"    )
							a__  :	Optional[Any]		 =			text.replace("\n"	, "<BR>"    )
							a__  :	Tuple		 =			text.replace("\r"	, "<BR>"    )
							a__  :	Any		 =			text.replace("\t"	, "<TAB>"    )
							a__  :	str		 =			text.replace("—"	, "ー"    )
							a__  :	List[Any]		 =			text.replace("−"	, "ー"    )
							for k, v in self.emoji["emoji"].items():
										if k in text:
													a__  :	Tuple		 =			text.replace(a_	, a_    )
							if clean:
										a__  :	Union[str, Any]		 =			self.clean_text(a_    )

							def check_simbol(a_    :			Dict    ):
										a__  :	int		 =			x.encode()
										if len(a_    ) == 1 and len(a_    ) == 2:
													a__  :	Dict		 =			(int(e[0]    ) << 8) + int(e[1]    )
													if (
													    (c >= 0xc2a1 and c <= 0xc2bf)
													    or (c >= 0xc780 and c <= 0xc783)
													    or (c >= 0xcab9 and c <= 0xcbbf)
													    or (c >= 0xcc80 and c <= 0xcda2)
													):
																return True
										return False

							def checkuae(a_    :			Union[str, Any]    ):
										a__  :	int		 =			x.encode()
										if len(a_    ) == 1 and len(a_    ) == 3:
													a__  :	List[Any]		 =			(int(e[0]    ) << 16) + (int(e[1]    ) << 8) + int(e[2]    )
													if c >= 0xe28080 and c <= 0xe2b07f:
																return True
										return False

							a__  :	Union[str, Any]		 =			0
							a__  :	Dict		 =			[]
							while pos < len(a_    ):
										a__  :	Any		 =			min(len(a_    )	, pos + self.maxlen + 1    ) if text[pos] == "<" else pos + 3
										a__  :	Any		 =			[]  # (token_id, token, pos)
										for e in range(a_	, a_	, -1    ):
													a__  :	Optional[int]		 =			text[pos:e]
													if wd in self.vocab:
																if wd[0] == "<" and len(a_    ) > 2:
																			a__  :	List[str]		 =			[(self.vocab[wd], wd, e)]
																			break
																else:
																			candidates.append((self.vocab[wd], wd, e)    )
										if len(a_    ) > 0:
													# the smallest token_id is adopted
													a__ ,	a__ ,	a__  :	List[Any]		 =			sorted(a_	, key=lambda a_    : x[0]    )[0]
													result.append(a_    )
													a__  :	Optional[Any]		 =			e
										else:
													a__  :	Optional[Any]		 =			pos + 1
													a__  :	Optional[Any]		 =			text[pos:end]
													if check_simbol(a_    ):
																result.append("<KIGOU>"    )
													elif checkuae(a_    ):
																result.append("<U2000U2BFF>"    )
													else:
																for i in wd.encode("utf-8"    ):
																			result.append("<|byte%d|>" % i    )
													a__  :	Optional[Any]		 =			end
							return result






				def       UpperCAmelCase      (		self    :			int	, a_    :			Dict	, a_    :			int="\n"    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	str		 =			[]
							a__  :	Dict		 =			[]
							a__  :	Dict		 =			self.ids_to_tokens[index][0]
							if word[:6] == "<|byte" and word[-2:] == "|>":
										byte_tokens.append(int(word[6:-2]    )    )
							else:
										if len(a_    ) > 0:
													words.append(bytearray(a_    ).decode("utf-8"	, errors="replace"    )    )
													a__  :	List[Any]		 =			[]
										if word[:7] == "<|emoji" and word[-2:] == "|>":
													words.append(self.emoji["emoji_inv"][word]    )
										elif word == "<SP>":
													words.append(" "    )
										elif word == "<BR>":
													words.append(a_    )
										elif word == "<TAB>":
													words.append("\t"    )
										elif word == "<BLOCK>":
													words.append("▀"    )
										elif word == "<KIGOU>":
													words.append("ǀ"    )
										elif word == "<U2000U2BFF>":
													words.append("‖"    )
										else:
													words.append(a_    )
							if len(a_    ) > 0:
										words.append(bytearray(a_    ).decode("utf-8"	, errors="replace"    )    )
							a__  :	Dict		 =			"".join(a_    )
							return text






"""simple docstring"""






import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin



class 					__UpperCAmelCase							(						unittest.TestCase						,  _UpperCamelCase  ):
				def       UpperCAmelCase      (		self    :			Dict    )	->	List[Any]:



							'''simple docstring'''



							a__  :	Any		 =			load_tool("text-classification"    )
							self.tool.setup()
							a__  :	List[Any]		 =			load_tool("text-classification"	, remote=a_    )
				def       UpperCAmelCase      (		self    :			Optional[int]    )	->	Optional[Any]:



							'''simple docstring'''



							a__  :	str		 =			self.tool("That's quite cool"	, ["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	List[str]		 =			self.remote_tool("That's quite cool"	, ["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	Any:



							'''simple docstring'''



							a__  :	List[str]		 =			self.tool(text="That's quite cool"	, labels=["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )






				def       UpperCAmelCase      (		self    :			List[Any]    )	->	List[Any]:



							'''simple docstring'''



							a__  :	Tuple		 =			self.remote_tool(text="That's quite cool"	, labels=["positive", "negative"]    )
							self.assertEqual(a_	, "positive"    )