





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A				:    str       =   logging.get_logger(__name__)
__A				:    Tuple       =   {"vocab_file": "spiece.model"}

__A				:    Union[str, Any]       =   {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__A				:    Optional[int]       =   {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

__A				:    int       =   "▁"

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	def __init__(    self							,					_A							,					_A=True							,					_A=True							,					_A=False							,					_A="[CLS]"							,					_A="[SEP]"							,					_A="<unk>"							,					_A="[SEP]"							,					_A="<pad>"							,					_A="[CLS]"							,					_A="[MASK]"							,					_A = None							,					**_A							,					):






				'''simple docstring'''







				UpperCAmelCase   		=				(
				    AddedToken(_A							,					lstrip=_A							,					rstrip=_A							,					normalized=_A		)
				    if isinstance(_A							,					_A		)
				    else mask_token
				)

				UpperCAmelCase   		=				{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    do_lower_case=_A							,					remove_space=_A							,					keep_accents=_A							,					bos_token=_A							,					eos_token=_A							,					unk_token=_A							,					sep_token=_A							,					pad_token=_A							,					cls_token=_A							,					mask_token=_A							,					sp_model_kwargs=self.sp_model_kwargs							,					**_A							,					)

				UpperCAmelCase   		=				do_lower_case
				UpperCAmelCase   		=				remove_space
				UpperCAmelCase   		=				keep_accents
				UpperCAmelCase   		=				vocab_file

				UpperCAmelCase   		=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(_A		)
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				return len(self.sp_model		)
	def      _lowercase				(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				{self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab
	def __getstate__(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.__dict__.copy()
				UpperCAmelCase   		=				None
				return state
	def __setstate__(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				d

				# for backward compatibility
				if not hasattr(self							,					'''sp_model_kwargs'''		):
							UpperCAmelCase   		=				{}

				UpperCAmelCase   		=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(self.vocab_file		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				if self.remove_space:
							UpperCAmelCase   		=				''' '''.join(inputs.strip().split()		)
				else:
							UpperCAmelCase   		=				inputs
				UpperCAmelCase   		=				outputs.replace('''``'''							,					'''"'''		).replace('''\'\''''							,					'''"'''		)

				if not self.keep_accents:
							UpperCAmelCase   		=				unicodedata.normalize('''NFKD'''							,					_A		)
							UpperCAmelCase   		=				''''''.join([c for c in outputs if not unicodedata.combining(_A		)]		)
				if self.do_lower_case:
							UpperCAmelCase   		=				outputs.lower()

				return outputs
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.preprocess_text(_A		)
				UpperCAmelCase   		=				self.sp_model.encode(_A							,					out_type=_A		)
				UpperCAmelCase   		=				[]
				for piece in pieces:
							if len(_A		) > 1 and piece[-1] == str(''','''		) and piece[-2].isdigit():
										UpperCAmelCase   		=				self.sp_model.EncodeAsPieces(piece[:-1].replace(_A							,					''''''		)		)
										if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
													if len(cur_pieces[0]		) == 1:
																UpperCAmelCase   		=				cur_pieces[1:]
													else:
																UpperCAmelCase   		=				cur_pieces[0][1:]
										cur_pieces.append(piece[-1]		)
										new_pieces.extend(_A		)
							else:
										new_pieces.append(_A		)

				return new_pieces
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.sp_model.PieceToId(_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.sp_model.IdToPiece(_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				UpperCAmelCase   		=				''''''
				UpperCAmelCase   		=				False
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										if not prev_is_special:
													out_string += " "
										out_string += self.sp_model.decode(_A		) + token
										UpperCAmelCase   		=				True
										UpperCAmelCase   		=				[]
							else:
										current_sub_tokens.append(_A		)
										UpperCAmelCase   		=				False
				out_string += self.sp_model.decode(_A		)
				return out_string.strip()
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]
				if token_ids_a is None:
							return cls + token_ids_a + sep
				return cls + token_ids_a + sep + token_ids_a + sep
	def      _lowercase				(    self							,					_A							,					_A = None							,					_A = False		):






				'''simple docstring'''







				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_A							,					token_ids_a=_A							,					already_has_special_tokens=_A		)

				if token_ids_a is not None:
							return [1] + ([0] * len(_A		)) + [1] + ([0] * len(_A		)) + [1]
				return [1] + ([0] * len(_A		)) + [1]
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				if not os.path.isdir(_A		):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
							return
				UpperCAmelCase   		=				os.path.join(
				    _A							,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
							copyfile(self.vocab_file							,					_A		)
				elif not os.path.isfile(self.vocab_file		):
							with open(_A							,					'''wb'''		) as fi:
										UpperCAmelCase   		=				self.sp_model.serialized_model_proto()
										fi.write(_A		)

				return (out_vocab_file,)










import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
				from sagemaker import Session, TrainingJobAnalytics
				from sagemaker.huggingface import HuggingFace

@pytest.mark.skipif(
    literal_eval(os.getenv('''TEST_SAGEMAKER'''     ,							'''False'''			)			) is not True     ,							reason='''Skipping test because should only be run when releasing minor transformers version'''     ,							)
@pytest.mark.usefixtures('''sm_env'''			)
@parameterized_class(
    [
        {
            '''framework''': '''pytorch''',
            '''script''': '''run_glue.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_5_0, '''eval_accuracy''': 0.6, '''eval_loss''': 0.9},
        },
        {
            '''framework''': '''tensorflow''',
            '''script''': '''run_tf.py''',
            '''model_name_or_path''': '''distilbert-base-cased''',
            '''instance_type''': '''ml.g4dn.xlarge''',
            '''results''': {'''train_runtime''': 6_0_0, '''eval_accuracy''': 0.3, '''eval_loss''': 0.9},
        },
    ]			)
class 	A_       (unittest.TestCase			):
	def      _lowercase				(    self		):






				'''simple docstring'''







				if self.framework == "pytorch":
							subprocess.run(
							    F"""cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py""".split()							,					encoding='''utf-8'''							,					check=_A							,					)
				assert hasattr(self							,					'''env'''		)
	def      _lowercase				(    self							,					_A=1		):






				'''simple docstring'''







				return HuggingFace(
				    entry_point=self.script							,					source_dir=self.env.test_path							,					role=self.env.role							,					image_uri=self.env.image_uri							,					base_job_name=F"""{self.env.base_job_name}-single"""							,					instance_count=_A							,					instance_type=self.instance_type							,					debugger_hook_config=_A							,					hyperparameters={**self.env.hyperparameters, '''model_name_or_path''': self.model_name_or_path}							,					metric_definitions=self.env.metric_definitions							,					py_version='''py36'''							,					)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				TrainingJobAnalytics(_A		).export_csv(F"""{self.env.test_path}/{job_name}_metrics.csv"""		)







	def      _lowercase				(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.create_estimator()

				# run training
				estimator.fit()

				# result dataframe
				UpperCAmelCase   		=				TrainingJobAnalytics(estimator.latest_training_job.name		).dataframe()

				# extract kpis
				UpperCAmelCase   		=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_accuracy''']['''value''']		)
				UpperCAmelCase   		=				list(result_metrics_df[result_metrics_df.metric_name == '''eval_loss''']['''value''']		)
				# get train time from SageMaker job, this includes starting, preprocessing, stopping
				UpperCAmelCase   		=				(
				    Session().describe_training_job(estimator.latest_training_job.name		).get('''TrainingTimeInSeconds'''							,					9_9_9_9_9_9		)
				)

				# assert kpis
				assert train_runtime <= self.results["train_runtime"]
				assert all(t >= self.results['''eval_accuracy'''] for t in eval_accuracy		)
				assert all(t <= self.results['''eval_loss'''] for t in eval_loss		)

				# dump tests result into json file to share in PR
				with open(F"""{estimator.latest_training_job.name}.json"""							,					'''w'''		) as outfile:
							json.dump({'''train_time''': train_runtime, '''eval_accuracy''': eval_accuracy, '''eval_loss''': eval_loss}							,					_A		)



