






"""simple docstring"""





import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
  import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE					:				Tuple					    =						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					:				Optional[Any]					    =						{"vocab_file": "vocab.txt"}

_SCREAMING_SNAKE_CASE					:				Tuple					    =						{
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

_SCREAMING_SNAKE_CASE					:				int					    =						{
    "openbmb/cpm-ant-10b": 1024,
}







def      lowerCamelCase__       (							_lowerCamelCase       :		Optional[Any] )      ->				Optional[int]:
  lowerCamelCase_         =						collections.OrderedDict()
  with open(__UpperCamelCase		,				'r'		,				encoding='utf-8' ) as reader:
    lowerCamelCase_         =						reader.readlines()
  for index, token in enumerate(__UpperCamelCase ):
    lowerCamelCase_         =						token.rstrip('\n' )
    lowerCamelCase_         =						index
  return vocab






class      a						(					SCREAMING_SNAKE_CASE__		):






    def __init__(       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							int="<unk>"       ,						__SCREAMING_SNAKE_CASE     :							List[str]=200					)     ->		int:
      lowerCamelCase_         =						vocab
      lowerCamelCase_         =						unk_token
      lowerCamelCase_         =						max_input_chars_per_word






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Any					)     ->		List[Any]:
      lowerCamelCase_         =						list(__SCREAMING_SNAKE_CASE					)
      if len(__SCREAMING_SNAKE_CASE					) > self.max_input_chars_per_word:
        return [self.unk_token]

      lowerCamelCase_         =						0
      lowerCamelCase_         =						[]
      while start < len(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						len(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						None
        while start < end:
          lowerCamelCase_         =						''.join(chars[start:end]					)
          if substr in self.vocab:
            lowerCamelCase_         =						substr
            break
          end -= 1
        if cur_substr is None:
          sub_tokens.append(self.unk_token					)
          start += 1
        else:
          sub_tokens.append(__SCREAMING_SNAKE_CASE					)
          lowerCamelCase_         =						end

      return sub_tokens






class      a						(					SCREAMING_SNAKE_CASE__		):
    SCREAMING_SNAKE_CASE						:     str						= VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						:     str						= PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE						:     Any						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE						:     Optional[int]						= ["""input_ids""", """attention_mask"""]
    SCREAMING_SNAKE_CASE						:     str						= False






    def __init__(       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Dict       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]="<d>"       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]="</d>"       ,						__SCREAMING_SNAKE_CASE     :							str="<s>"       ,						__SCREAMING_SNAKE_CASE     :							List[str]="</s>"       ,						__SCREAMING_SNAKE_CASE     :							str="<pad>"       ,						__SCREAMING_SNAKE_CASE     :							List[str]="<unk>"       ,						__SCREAMING_SNAKE_CASE     :							List[str]="</n>"       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]="</_>"       ,						__SCREAMING_SNAKE_CASE     :							str="left"       ,						**__SCREAMING_SNAKE_CASE     :							Any       ,						)     ->		Any:
      requires_backends(self       ,						['jieba']					)
      super().__init__(
          bod_token=__SCREAMING_SNAKE_CASE       ,						eod_token=__SCREAMING_SNAKE_CASE       ,						bos_token=__SCREAMING_SNAKE_CASE       ,						eos_token=__SCREAMING_SNAKE_CASE       ,						pad_token=__SCREAMING_SNAKE_CASE       ,						unk_token=__SCREAMING_SNAKE_CASE       ,						line_token=__SCREAMING_SNAKE_CASE       ,						space_token=__SCREAMING_SNAKE_CASE       ,						padding_side=__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE       ,						)
      lowerCamelCase_         =						bod_token
      lowerCamelCase_         =						eod_token
      lowerCamelCase_         =						load_vocab(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						self.encoder[space_token]
      lowerCamelCase_         =						self.encoder[line_token]

      del self.encoder[space_token]
      del self.encoder[line_token]

      lowerCamelCase_         =						collections.OrderedDict(sorted(self.encoder.items()       ,						key=lambda __SCREAMING_SNAKE_CASE					: x[1]					)					)
      lowerCamelCase_         =						{v: k for k, v in self.encoder.items()}

      lowerCamelCase_         =						WordpieceTokenizer(vocab=self.encoder       ,						unk_token=self.unk_token					)






    @property
    def    UpperCamelCase       (       self     :							Tuple					)     ->		int:
      return self.encoder[self.bod_token]






    @property
    def    UpperCamelCase       (       self     :							Dict					)     ->		List[str]:
      return self.encoder[self.eod_token]






    @property
    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Dict:
      return self.encoder["\n"]






    @property
    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		int:
      return len(self.encoder					)






    def    UpperCamelCase       (       self     :							str					)     ->		Union[str, Any]:
      return dict(self.encoder       ,						**self.added_tokens_encoder					)






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]					)     ->		Dict:
      lowerCamelCase_         =						[]
      for x in jieba.cut(__SCREAMING_SNAKE_CASE       ,						cut_all=__SCREAMING_SNAKE_CASE					):
        output_tokens.extend(self.wordpiece_tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)					)
      return output_tokens






    def    UpperCamelCase       (       self     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							Dict       ,						**__SCREAMING_SNAKE_CASE     :							Dict					)     ->		Any:
      lowerCamelCase_         =						[i for i in token_ids if i >= 0]
      lowerCamelCase_         =						[
          x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
      ]
      return super()._decode(__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Dict       ,						__SCREAMING_SNAKE_CASE     :							List[str]					)     ->		str:
      return token in self.encoder






    def    UpperCamelCase       (       self     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							List[str]					)     ->		str:
      return "".join(__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[str]					)     ->		Optional[int]:
      return self.encoder.get(__SCREAMING_SNAKE_CASE       ,						self.encoder.get(self.unk_token					)					)






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							str					)     ->		int:
      return self.decoder.get(__SCREAMING_SNAKE_CASE       ,						self.unk_token					)






    def    UpperCamelCase       (       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[str] = None					)     ->		Tuple[str]:
      if os.path.isdir(__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						os.path.join(
            __SCREAMING_SNAKE_CASE       ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
      else:
        lowerCamelCase_         =						(filename_prefix + '-' if filename_prefix else '') + save_directory
      lowerCamelCase_         =						0
      if " " in self.encoder:
        lowerCamelCase_         =						self.encoder[' ']
        del self.encoder[" "]
      if "\n" in self.encoder:
        lowerCamelCase_         =						self.encoder['\n']
        del self.encoder["\n"]
      lowerCamelCase_         =						collections.OrderedDict(sorted(self.encoder.items()       ,						key=lambda __SCREAMING_SNAKE_CASE					: x[1]					)					)
      with open(__SCREAMING_SNAKE_CASE       ,						'w'       ,						encoding='utf-8'					) as writer:
        for token, token_index in self.encoder.items():
          if index != token_index:
            logger.warning(
                F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                ' Please check that the vocabulary is not corrupted!'					)
            lowerCamelCase_         =						token_index
          writer.write(token + '\n'					)
          index += 1
      return (vocab_file,)






    def    UpperCamelCase       (       self     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							List[int] = None					)     ->		List[int]:
      if token_ids_a is None:
        return [self.bos_token_id] + token_ids_a
      return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[List[int]] = None       ,						__SCREAMING_SNAKE_CASE     :							bool = False					)     ->		List[int]:

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__SCREAMING_SNAKE_CASE       ,						token_ids_a=__SCREAMING_SNAKE_CASE       ,						already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

      if token_ids_a is not None:
        return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE					))
      return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					))





import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


A   :     Any       =				logging.getLogger(__name__)


def    a__       (						__UpperCamelCase   ,					__UpperCamelCase ):
      return (preds == labels).mean()


@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(
      metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}     ,					)



@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(metadata={'''help''': '''The name of the task to train on: ''' + ''', '''.join(processors.keys()    )}    )
  lowerCamelCase__   =  field(metadata={'''help''': '''Should contain the data files for the task.'''}    )
  lowerCamelCase__   =  field(
      default=1_2_8     ,					metadata={
          '''help''': (
              '''The maximum total input sequence length after tokenization. Sequences longer '''
              '''than this will be truncated, sequences shorter will be padded.'''
          )
      }     ,					)
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}    )



def    a__       (						):
      # See all possible arguments in src/transformers/training_args.py
      # or by passing the --help flag to this script.
      # We now keep distinct sets of args, for a cleaner separation of concerns.

      SCREAMING_SNAKE_CASE_      =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )
      SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_      =     parser.parse_args_into_dataclasses()

      if (
          os.path.exists(training_args.output_dir )
          and os.listdir(training_args.output_dir )
          and training_args.do_train
          and not training_args.overwrite_output_dir
      ):
            raise ValueError(
                F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
                " --overwrite_output_dir to overcome." )

      # Setup logging
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,					datefmt="%m/%d/%Y %H:%M:%S"   ,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   ,					)
      logger.warning(
          "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"   ,					training_args.local_rank   ,					training_args.device   ,					training_args.n_gpu   ,					bool(training_args.local_rank != -1 )   ,					training_args.fpaa   ,					)
      # Set the verbosity to info of the Transformers logger (on main process only):
      if is_main_process(training_args.local_rank ):
            transformers.utils.logging.set_verbosity_info()
            transformers.utils.logging.enable_default_handler()
            transformers.utils.logging.enable_explicit_format()
      logger.info("Training/evaluation parameters %s"   ,					__UpperCamelCase )

      # Set seed
      set_seed(training_args.seed )

      try:
            SCREAMING_SNAKE_CASE_      =     processors[data_args.task_name]()
            SCREAMING_SNAKE_CASE_      =     processor.get_labels()
            SCREAMING_SNAKE_CASE_      =     len(__UpperCamelCase )
      except KeyError:
            raise ValueError("Task not found: %s" % (data_args.task_name) )

      # Load pretrained model and tokenizer
      #
      # Distributed training:
      # The .from_pretrained methods guarantee that only one local process can concurrently
      # download model & vocab.

      SCREAMING_SNAKE_CASE_      =     AutoConfig.from_pretrained(
          model_args.config_name if model_args.config_name else model_args.model_name_or_path   ,					num_labels=__UpperCamelCase   ,					finetuning_task=data_args.task_name   ,					cache_dir=model_args.cache_dir   ,					)
      SCREAMING_SNAKE_CASE_      =     AutoTokenizer.from_pretrained(
          model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   ,					cache_dir=model_args.cache_dir   ,					)
      SCREAMING_SNAKE_CASE_      =     AutoModelForMultipleChoice.from_pretrained(
          model_args.model_name_or_path   ,					from_tf=bool(".ckpt" in model_args.model_name_or_path )   ,					config=__UpperCamelCase   ,					cache_dir=model_args.cache_dir   ,					)

      # Get datasets
      SCREAMING_SNAKE_CASE_      =     (
          MultipleChoiceDataset(
              data_dir=data_args.data_dir   ,					tokenizer=__UpperCamelCase   ,					task=data_args.task_name   ,					max_seq_length=data_args.max_seq_length   ,					overwrite_cache=data_args.overwrite_cache   ,					mode=Split.train   ,					)
          if training_args.do_train
          else None
      )
      SCREAMING_SNAKE_CASE_      =     (
          MultipleChoiceDataset(
              data_dir=data_args.data_dir   ,					tokenizer=__UpperCamelCase   ,					task=data_args.task_name   ,					max_seq_length=data_args.max_seq_length   ,					overwrite_cache=data_args.overwrite_cache   ,					mode=Split.dev   ,					)
          if training_args.do_eval
          else None
      )

      def compute_metrics(__UpperCamelCase ) -> Dict:
            SCREAMING_SNAKE_CASE_      =     np.argmax(p.predictions   ,					axis=1 )
            return {"acc": simple_accuracy(__UpperCamelCase   ,					p.label_ids )}

      # Data collator
      SCREAMING_SNAKE_CASE_      =     DataCollatorWithPadding(__UpperCamelCase   ,					pad_to_multiple_of=8 ) if training_args.fpaa else None

      # Initialize our Trainer
      SCREAMING_SNAKE_CASE_      =     Trainer(
          model=__UpperCamelCase   ,					args=__UpperCamelCase   ,					train_dataset=__UpperCamelCase   ,					eval_dataset=__UpperCamelCase   ,					compute_metrics=__UpperCamelCase   ,					data_collator=__UpperCamelCase   ,					)

      # Training
      if training_args.do_train:
            trainer.train(
                model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path ) else None )
            trainer.save_model()
            # For convenience, we also re-save the tokenizer to the same directory,
            # so that you can share your model easily on huggingface.co/models =)
            if trainer.is_world_master():
                  tokenizer.save_pretrained(training_args.output_dir )

    # Evaluation
      SCREAMING_SNAKE_CASE_      =     {}
      if training_args.do_eval:
            logger.info("*** Evaluate ***" )

            SCREAMING_SNAKE_CASE_      =     trainer.evaluate()

            SCREAMING_SNAKE_CASE_      =     os.path.join(training_args.output_dir   ,					"eval_results.txt" )
            if trainer.is_world_master():
                  with open(__UpperCamelCase   ,					"w" ) as writer:
                        logger.info("***** Eval results *****" )
                        for key, value in result.items():
                              logger.info("  %s = %s"   ,					__UpperCamelCase   ,					__UpperCamelCase )
                              writer.write("%s = %s\n" % (key, value) )

                        results.update(__UpperCamelCase )

      return results


def    a__       (						__UpperCamelCase ):
      # For xla_spawn (TPUs)
      main()


if __name__ == "__main__":
      main()


