




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
					import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__    			=			logging.get_logger(__name__)

UpperCamelCase__    			=			{"""vocab_file""": """vocab.txt"""}

UpperCamelCase__    			=			{
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

UpperCamelCase__    			=			{
    """openbmb/cpm-ant-10b""": 1024,
}







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Optional[Any]     ):
					__lowerCAmelCase				   =       collections.OrderedDict()
					with open(SCREAMING_SNAKE_CASE_      ,      "r"      ,      encoding="utf-8"     ) as reader:
										__lowerCAmelCase				   =       reader.readlines()
					for index, token in enumerate(SCREAMING_SNAKE_CASE_     ):
										__lowerCAmelCase				   =       token.rstrip("\n"     )
										__lowerCAmelCase				   =       index
					return vocab
class 			a__    (					snake_case__      ):






						def __init__(   self		,			_A		,			_A="<unk>"		,			_A=2_0_0			):




											"""simple docstring"""



											__lowerCAmelCase				   =       vocab
											__lowerCAmelCase				   =       unk_token
											__lowerCAmelCase				   =       max_input_chars_per_word






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(_A			)
											if len(_A			) > self.max_input_chars_per_word:
																return [self.unk_token]

											__lowerCAmelCase				   =       0
											__lowerCAmelCase				   =       []
											while start < len(_A			):
																__lowerCAmelCase				   =       len(_A			)
																__lowerCAmelCase				   =       None
																while start < end:
																					__lowerCAmelCase				   =       "".join(chars[start:end]			)
																					if substr in self.vocab:
																										__lowerCAmelCase				   =       substr
																										break
																					end -= 1
																if cur_substr is None:
																					sub_tokens.append(self.unk_token			)
																					start += 1
																else:
																					sub_tokens.append(_A			)
																					__lowerCAmelCase				   =       end

											return sub_tokens






class 			a__    (					snake_case__      ):
						_a							:    Any					=    VOCAB_FILES_NAMES
						_a							:    str					=    PRETRAINED_VOCAB_FILES_MAP
						_a							:    Optional[Any]					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:    Any					=    ["""input_ids""", """attention_mask"""]
						_a							:    int					=    False






						def __init__(   self		,			_A		,			_A="<d>"		,			_A="</d>"		,			_A="<s>"		,			_A="</s>"		,			_A="<pad>"		,			_A="<unk>"		,			_A="</n>"		,			_A="</_>"		,			_A="left"		,			**_A		,			):




											"""simple docstring"""



											requires_backends(self		,			["jieba"]			)
											super().__init__(
											    bod_token=_A		,			eod_token=_A		,			bos_token=_A		,			eos_token=_A		,			pad_token=_A		,			unk_token=_A		,			line_token=_A		,			space_token=_A		,			padding_side=_A		,			**_A		,			)
											__lowerCAmelCase				   =       bod_token
											__lowerCAmelCase				   =       eod_token
											__lowerCAmelCase				   =       load_vocab(_A			)
											__lowerCAmelCase				   =       self.encoder[space_token]
											__lowerCAmelCase				   =       self.encoder[line_token]

											del self.encoder[space_token]
											del self.encoder[line_token]

											__lowerCAmelCase				   =       collections.OrderedDict(sorted(self.encoder.items()		,			key=lambda _A			: x[1]			)			)
											__lowerCAmelCase				   =       {v: k for k, v in self.encoder.items()}

											__lowerCAmelCase				   =       WordpieceTokenizer(vocab=self.encoder		,			unk_token=self.unk_token			)






						@property
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return self.encoder[self.bod_token]






						@property
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return self.encoder[self.eod_token]






						@property
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return self.encoder["\n"]






						@property
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return len(self.encoder			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											return dict(self.encoder		,			**self.added_tokens_encoder			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       []
											for x in jieba.cut(_A		,			cut_all=_A			):
																output_tokens.extend(self.wordpiece_tokenizer.tokenize(_A			)			)
											return output_tokens






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       [i for i in token_ids if i >= 0]
											__lowerCAmelCase				   =       [
											    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
											]
											return super()._decode(_A		,			**_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return token in self.encoder






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return "".join(_A			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return self.encoder.get(_A		,			self.encoder.get(self.unk_token			)			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A			):




											"""simple docstring"""



											return self.decoder.get(_A		,			self.unk_token			)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None			):




											"""simple docstring"""



											if os.path.isdir(_A			):
																__lowerCAmelCase				   =       os.path.join(
																    _A		,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
											else:
																__lowerCAmelCase				   =       (filename_prefix + "-" if filename_prefix else "") + save_directory
											__lowerCAmelCase				   =       0
											if " " in self.encoder:
																__lowerCAmelCase				   =       self.encoder[" "]
																del self.encoder[" "]
											if "\n" in self.encoder:
																__lowerCAmelCase				   =       self.encoder["\n"]
																del self.encoder["\n"]
											__lowerCAmelCase				   =       collections.OrderedDict(sorted(self.encoder.items()		,			key=lambda _A			: x[1]			)			)
											with open(_A		,			"w"		,			encoding="utf-8"			) as writer:
																for token, token_index in self.encoder.items():
																					if index != token_index:
																										logger.warning(
																										    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																										    " Please check that the vocabulary is not corrupted!"			)
																										__lowerCAmelCase				   =       token_index
																					writer.write(token + "\n"			)
																					index += 1
											return (vocab_file,)






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None			):




											"""simple docstring"""



											if token_ids_a is None:
																return [self.bos_token_id] + token_ids_a
											return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A = None		,			_A = False			):




											"""simple docstring"""



											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_A		,			token_ids_a=_A		,			already_has_special_tokens=_A			)

											if token_ids_a is not None:
																return [1] + ([0] * len(_A			)) + [1] + ([0] * len(_A			))
											return [1] + ([0] * len(_A			))










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm
					import torch_xla.debug.metrics as met
class 			a__    (					snake_case__      ):






						def __init__(   self		,			*_A		,			_A=None		,			_A=None		,			**_A			):




											"""simple docstring"""



											super().__init__(*_A		,			**_A			)
											__lowerCAmelCase				   =       eval_examples
											__lowerCAmelCase				   =       post_process_function






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A = None		,			_A=None		,			_A = None		,			_A = "eval"		,			**_A		,			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()
											__lowerCAmelCase				   =       (
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"			) is not None else self.args.generation_max_length
											)
											__lowerCAmelCase				   =       (
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"			) is not None else self.args.generation_num_beams
											)
											__lowerCAmelCase				   =       gen_kwargs

											__lowerCAmelCase				   =       self.eval_dataset if eval_dataset is None else eval_dataset
											__lowerCAmelCase				   =       self.get_eval_dataloader(_A			)
											__lowerCAmelCase				   =       self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Evaluation"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics
											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A			)
																__lowerCAmelCase				   =       self.compute_metrics(_A			)

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()			):
																					if not key.startswith(f"""{metric_key_prefix}_"""			):
																										__lowerCAmelCase				   =       metrics.pop(_A			)

																metrics.update(output.metrics			)
											else:
																__lowerCAmelCase				   =       output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(_A			)

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()			)

											__lowerCAmelCase				   =       self.callback_handler.on_evaluate(self.args		,			self.state		,			self.control		,			_A			)
											return metrics






						def 				__SCREAMING_SNAKE_CASE(   self		,			_A		,			_A		,			_A=None		,			_A = "test"		,			**_A			):




											"""simple docstring"""



											__lowerCAmelCase				   =       gen_kwargs.copy()

											__lowerCAmelCase				   =       self.get_test_dataloader(_A			)

											# Temporarily disable metric computation, we will do it in the loop here.
											__lowerCAmelCase				   =       self.compute_metrics
											__lowerCAmelCase				   =       None
											__lowerCAmelCase				   =       time.time()
											__lowerCAmelCase				   =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__lowerCAmelCase				   =       eval_loop(
																    _A		,			description="Prediction"		,			prediction_loss_only=True if compute_metrics is None else None		,			ignore_keys=_A		,			metric_key_prefix=_A		,			)
											finally:
																__lowerCAmelCase				   =       compute_metrics

											__lowerCAmelCase				   =       self.args.eval_batch_size * self.args.world_size
											if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
											output.metrics.update(
											    speed_metrics(
											        _A		,			_A		,			num_samples=output.num_samples		,			num_steps=math.ceil(output.num_samples / total_batch_size			)		,			)			)
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__lowerCAmelCase				   =       self.post_process_function(_A		,			_A		,			_A		,			"predict"			)
											__lowerCAmelCase				   =       self.compute_metrics(_A			)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()			):
																if not key.startswith(f"""{metric_key_prefix}_"""			):
																					__lowerCAmelCase				   =       metrics.pop(_A			)
											metrics.update(output.metrics			)
											return PredictionOutput(predictions=predictions.predictions		,			label_ids=predictions.label_ids		,			metrics=_A			)




