





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


__A				:    List[Any]       =   logging.get_logger(__name__)

__A				:    str       =   {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

__A				:    Optional[int]       =   {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

__A				:    List[str]       =   {
    "abeja/gpt-neox-japanese-2.7b": 2_048,
}





def 						__SCREAMING_SNAKE_CASE  (				UpperCamelCase__			,			UpperCamelCase__ )  ->				Tuple:







			'''simple docstring'''

			with open(UpperCamelCase__			,			'''r'''			,			encoding='''utf-8''' ) as f:
						UpperCAmelCase   		=				json.loads(f.read() )

			UpperCAmelCase   		=				collections.OrderedDict()
			UpperCAmelCase   		=				collections.OrderedDict()
			UpperCAmelCase   		=				collections.OrderedDict()
			with open(UpperCamelCase__			,			'''r'''			,			encoding='''utf-8''' ) as f:
						UpperCAmelCase   		=				f.readlines()
			UpperCAmelCase   		=				[[t.rstrip('''\n''' )] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n''' ).split(''',''' ) for t in token]
			for idx, b in enumerate(UpperCamelCase__ ):
						UpperCAmelCase   		=				b
						UpperCAmelCase   		=				idx
						for wd in b:
									UpperCAmelCase   		=				idx

			return vocab, raw_vocab, ids_to_tokens, emoji

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase__					=			['''input_ids''', '''attention_mask''']
	def __init__(    self							,					_A							,					_A							,					_A="<|endoftext|>"							,					_A="<|endoftext|>"							,					_A="<|startoftext|>"							,					_A="<|endoftext|>"							,					_A=False							,					**_A							,					):






				'''simple docstring'''







				super().__init__(
				    unk_token=_A							,					pad_token=_A							,					bos_token=_A							,					eos_token=_A							,					do_clean_text=_A							,					**_A							,					)
				if not os.path.isfile(_A		):
							raise ValueError(
							    F"""Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"""
							    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''		)
				if not os.path.isfile(_A		):
							raise ValueError(
							    F"""Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"""
							    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''		)
				UpperCAmelCase   		=				do_clean_text
				UpperCAmelCase		,			UpperCAmelCase		,			UpperCAmelCase		,			UpperCAmelCase   		=				load_vocab_and_emoji(_A							,					_A		)
				UpperCAmelCase   		=				SubWordJapaneseTokenizer(
				    vocab=self.vocab							,					ids_to_tokens=self.ids_to_tokens							,					emoji=self.emoji		)
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				return len(self.raw_vocab		)
	def      _lowercase				(    self		):






				'''simple docstring'''







				return dict(self.raw_vocab							,					**self.added_tokens_encoder		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.subword_tokenizer.tokenize(_A							,					clean=self.do_clean_text		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.vocab.get(_A							,					self.vocab.get(self.unk_token		)		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.subword_tokenizer.convert_id_to_token(_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				''''''.join(_A		).strip()
				return out_string
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				for is_user, text in conversation.iter_texts():
							input_ids.extend(self.encode(_A							,					add_special_tokens=_A		) + [self.eos_token_id]		)

				if len(_A		) > self.model_max_length:
							UpperCAmelCase   		=				input_ids[-self.model_max_length :]
				return input_ids







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				0
				if os.path.isdir(_A		):
							UpperCAmelCase   		=				os.path.join(
							    _A							,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
							UpperCAmelCase   		=				os.path.join(
							    _A							,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']		)
				else:
							UpperCAmelCase   		=				(
							    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
							)
							UpperCAmelCase   		=				(
							    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
							)
				with open(_A							,					'''w'''							,					encoding='''utf-8'''		) as writer:
							for token_index, token in self.ids_to_tokens.items():
										if index != token_index:
													logger.warning(
													    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
													    ''' Please check that the vocabulary is not corrupted!'''		)
													UpperCAmelCase   		=				token_index
										writer.write(''','''.join(_A		) + '''\n'''		)
										index += 1
				with open(_A							,					'''w'''							,					encoding='''utf-8'''		) as writer:
							json.dump(self.emoji							,					_A		)
				return vocab_file, emoji_file

class 	A_       (a_			):
	def __init__(    self							,					_A							,					_A							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				vocab  # same as swe
				UpperCAmelCase   		=				ids_to_tokens  # same as bpe
				UpperCAmelCase   		=				emoji
				UpperCAmelCase   		=				np.max([len(_A		) for w in self.vocab.keys()]		)
				UpperCAmelCase   		=				re.compile(r'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''		)
				UpperCAmelCase   		=				re.compile(r'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''		)
				UpperCAmelCase   		=				re.compile(r'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''		)
				UpperCAmelCase   		=				re.compile(
				    r'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''		)
				UpperCAmelCase   		=				re.compile(
				    r'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''		)
				UpperCAmelCase   		=				re.compile(
				    r'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''		)
				UpperCAmelCase   		=				'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
				UpperCAmelCase   		=				'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
				UpperCAmelCase   		=				str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}		)
	def __len__(    self		):






				'''simple docstring'''







				return len(self.ids_to_tokens		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.content_repattera.sub('''<URL>'''							,					_A		)
				UpperCAmelCase   		=				self.content_repattera.sub('''<EMAIL>'''							,					_A		)
				UpperCAmelCase   		=				self.content_repattera.sub('''<TEL>'''							,					_A		)
				UpperCAmelCase   		=				self.content_repattera.sub('''<DATE>'''							,					_A		)
				UpperCAmelCase   		=				self.content_repattera.sub('''<DATE>'''							,					_A		)
				UpperCAmelCase   		=				self.content_repattera.sub('''<PRICE>'''							,					_A		)
				UpperCAmelCase   		=				content.translate(self.content_transa		)
				while "<BLOCK><BLOCK>" in content:
							UpperCAmelCase   		=				content.replace('''<BLOCK><BLOCK>'''							,					'''<BLOCK>'''		)
				return content
	def      _lowercase				(    self							,					_A							,					_A=False		):






				'''simple docstring'''







				UpperCAmelCase   		=				text.replace(''' '''							,					'''<SP>'''		)
				UpperCAmelCase   		=				text.replace('''　'''							,					'''<SP>'''		)
				UpperCAmelCase   		=				text.replace('''\r\n'''							,					'''<BR>'''		)
				UpperCAmelCase   		=				text.replace('''\n'''							,					'''<BR>'''		)
				UpperCAmelCase   		=				text.replace('''\r'''							,					'''<BR>'''		)
				UpperCAmelCase   		=				text.replace('''\t'''							,					'''<TAB>'''		)
				UpperCAmelCase   		=				text.replace('''—'''							,					'''ー'''		)
				UpperCAmelCase   		=				text.replace('''−'''							,					'''ー'''		)
				for k, v in self.emoji["emoji"].items():
							if k in text:
										UpperCAmelCase   		=				text.replace(_A							,					_A		)
				if clean:
							UpperCAmelCase   		=				self.clean_text(_A		)

				def check_simbol(_A		):
							UpperCAmelCase   		=				x.encode()
							if len(_A		) == 1 and len(_A		) == 2:
										UpperCAmelCase   		=				(int(e[0]		) << 8) + int(e[1]		)
										if (
										    (c >= 0xc2a1 and c <= 0xc2bf)
										    or (c >= 0xc780 and c <= 0xc783)
										    or (c >= 0xcab9 and c <= 0xcbbf)
										    or (c >= 0xcc80 and c <= 0xcda2)
										):
													return True
							return False

				def checkuae(_A		):
							UpperCAmelCase   		=				x.encode()
							if len(_A		) == 1 and len(_A		) == 3:
										UpperCAmelCase   		=				(int(e[0]		) << 1_6) + (int(e[1]		) << 8) + int(e[2]		)
										if c >= 0xe28080 and c <= 0xe2b07f:
													return True
							return False

				UpperCAmelCase   		=				0
				UpperCAmelCase   		=				[]
				while pos < len(_A		):
							UpperCAmelCase   		=				min(len(_A		)							,					pos + self.maxlen + 1		) if text[pos] == '''<''' else pos + 3
							UpperCAmelCase   		=				[]  # (token_id, token, pos)
							for e in range(_A							,					_A							,					-1		):
										UpperCAmelCase   		=				text[pos:e]
										if wd in self.vocab:
													if wd[0] == "<" and len(_A		) > 2:
																UpperCAmelCase   		=				[(self.vocab[wd], wd, e)]
																break
													else:
																candidates.append((self.vocab[wd], wd, e)		)
							if len(_A		) > 0:
										# the smallest token_id is adopted
										UpperCAmelCase		,			UpperCAmelCase		,			UpperCAmelCase   		=				sorted(_A							,					key=lambda _A		: x[0]		)[0]
										result.append(_A		)
										UpperCAmelCase   		=				e
							else:
										UpperCAmelCase   		=				pos + 1
										UpperCAmelCase   		=				text[pos:end]
										if check_simbol(_A		):
													result.append('''<KIGOU>'''		)
										elif checkuae(_A		):
													result.append('''<U2000U2BFF>'''		)
										else:
													for i in wd.encode('''utf-8'''		):
																result.append('''<|byte%d|>''' % i		)
										UpperCAmelCase   		=				end
				return result







	def      _lowercase				(    self							,					_A							,					_A="\n"		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				UpperCAmelCase   		=				[]
				UpperCAmelCase   		=				self.ids_to_tokens[index][0]
				if word[:6] == "<|byte" and word[-2:] == "|>":
							byte_tokens.append(int(word[6:-2]		)		)
				else:
							if len(_A		) > 0:
										words.append(bytearray(_A		).decode('''utf-8'''							,					errors='''replace'''		)		)
										UpperCAmelCase   		=				[]
							if word[:7] == "<|emoji" and word[-2:] == "|>":
										words.append(self.emoji['''emoji_inv'''][word]		)
							elif word == "<SP>":
										words.append(''' '''		)
							elif word == "<BR>":
										words.append(_A		)
							elif word == "<TAB>":
										words.append('''\t'''		)
							elif word == "<BLOCK>":
										words.append('''▀'''		)
							elif word == "<KIGOU>":
										words.append('''ǀ'''		)
							elif word == "<U2000U2BFF>":
										words.append('''‖'''		)
							else:
										words.append(_A		)
				if len(_A		) > 0:
							words.append(bytearray(_A		).decode('''utf-8'''							,					errors='''replace'''		)		)
				UpperCAmelCase   		=				''''''.join(_A		)
				return text










import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
				from sagemaker import Session, TrainingJobAnalytics
				from sagemaker.huggingface import HuggingFace

@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''     ,							'''False'''			)			) is not True     ,							reason='''Skipping test because should only be run when releasing minor transformers version'''     ,							)
@pytest.mark.usefixtures('''sm_env'''			)
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_5_0, '''eval_accuracy''': 0.6, '''eval_loss''': 0.9},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_0_0, '''eval_accuracy''': 0.3, '''eval_loss''': 0.9},
        },
    ]			)
class 	A_       (unittest.TestCase			):
	def      _lowercase				(    self		):






				'''simple docstring'''







				if self.framework == "pytorch":
							subprocess.run(
							    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()							,					encoding='''utf-8'''							,					check=_A							,					)
				assert hasattr(self							,					'''env'''		)
	def      _lowercase				(    self							,					_A=1		):






				'''simple docstring'''







				return HuggingFace(
				    entry_point=self.script							,					source_dir=self.env.test_path							,					role=self.env.role							,					image_uri=self.env.image_uri							,					base_job_name=F"""{self.env.base_job_name}-single"""							,					instance_count=_A							,					instance_type=self.instance_type							,					debugger_hook_config=_A							,					hyperparameters={**self.env.hyperparameters, '''model_name_or_path''': self.model_name_or_path}							,					metric_definitions=self.env.metric_definitions							,					py_version='''py36'''							,					)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				TrainingJobAnalytics(_A		).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""		)







	def      _lowercase				(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.create_estimator()

				# run training
				estimator.fit()

				# result dataframe
				UpperCAmelCase   		=				TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

				# extract kpis
				UpperCAmelCase   		=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value''']		)
				UpperCAmelCase   		=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value''']		)
				# get train time from SageMaker job, this includes starting, preprocessing, stopping
				UpperCAmelCase   		=				(
				    Session().describe_training_job(estimator.latest_training_job.name		).get('''TrainingTimeInSeconds'''							,					9_9_9_9_9_9		)
				)

				# assert kpis
				assert train_runtime <= self.results["train_runtime"]
				assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy		)
				assert all(t <= self.results['''eval_loss'''] for t in eval_loss		)

				# dump tests result into json file to share in PR
				with open(F"""{estimator.latest_training_job.name}.json"""							,					'''w'''		) as outfile:
							json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}							,					_A		)



