




import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


UpperCamelCase__    			=			logging.getLogger(__name__)







def 	_a      (						SCREAMING_SNAKE_CASE_							:				int      ,      SCREAMING_SNAKE_CASE_							:				List[Any]     ):
					return (preds == labels).mean()
@dataclass
class 			a__    :
						_a							:    str					=    field(
						    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}      )
						_a							:    Optional[str]					=    field(
						    default=snake_case__ ,				metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}      )
						_a							:    Optional[str]					=    field(
						    default=snake_case__ ,				metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}      )
						_a							:    Optional[str]					=    field(
						    default=snake_case__ ,				metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""} ,				)






@dataclass
class 			a__    :
						_a							:    str					=    field(metadata={"""help""": """The name of the task to train on: """ + """, """.join(processors.keys()      )}      )
						_a							:    str					=    field(metadata={"""help""": """Should contain the data files for the task."""}      )
						_a							:    int					=    field(
						    default=1_2_8 ,				metadata={
						        """help""": (
						            """The maximum total input sequence length after tokenization. Sequences longer """
						            """than this will be truncated, sequences shorter will be padded."""
						        )
						    } ,				)
						_a							:    bool					=    field(
						    default=snake_case__ ,				metadata={"""help""": """Overwrite the cached training and evaluation sets"""}      )







def 	_a      (						):
					# See all possible arguments in src/transformers/training_args.py
					# or by passing the --help flag to this script.
					# We now keep distinct sets of args, for a cleaner separation of concerns.

					__lowerCAmelCase				   =       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )
					__lowerCAmelCase   ,							__lowerCAmelCase   ,							__lowerCAmelCase				   =       parser.parse_args_into_dataclasses()

					if (
					    os.path.exists(training_args.output_dir     )
					    and os.listdir(training_args.output_dir     )
					    and training_args.do_train
					    and not training_args.overwrite_output_dir
					):
										raise ValueError(
										    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
										    " --overwrite_output_dir to overcome."     )

					# Setup logging
					logging.basicConfig(
					    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"      ,      datefmt="%m/%d/%Y %H:%M:%S"      ,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN      ,      )
					logger.warning(
					    "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"      ,      training_args.local_rank      ,      training_args.device      ,      training_args.n_gpu      ,      bool(training_args.local_rank != -1     )      ,      training_args.fpaa      ,      )
					# Set the verbosity to info of the Transformers logger (on main process only):
					if is_main_process(training_args.local_rank     ):
										transformers.utils.logging.set_verbosity_info()
										transformers.utils.logging.enable_default_handler()
										transformers.utils.logging.enable_explicit_format()
					logger.info("Training/evaluation parameters %s"      ,      SCREAMING_SNAKE_CASE_     )

					# Set seed
					set_seed(training_args.seed     )

					try:
										__lowerCAmelCase				   =       processors[data_args.task_name]()
										__lowerCAmelCase				   =       processor.get_labels()
										__lowerCAmelCase				   =       len(SCREAMING_SNAKE_CASE_     )
					except KeyError:
										raise ValueError("Task not found: %s" % (data_args.task_name)     )

					# Load pretrained model and tokenizer
					#
					# Distributed training:
					# The .from_pretrained methods guarantee that only one local process can concurrently
					# download model & vocab.

					__lowerCAmelCase				   =       AutoConfig.from_pretrained(
					    model_args.config_name if model_args.config_name else model_args.model_name_or_path      ,      num_labels=SCREAMING_SNAKE_CASE_      ,      finetuning_task=data_args.task_name      ,      cache_dir=model_args.cache_dir      ,      )
					__lowerCAmelCase				   =       AutoTokenizer.from_pretrained(
					    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path      ,      cache_dir=model_args.cache_dir      ,      )
					__lowerCAmelCase				   =       AutoModelForMultipleChoice.from_pretrained(
					    model_args.model_name_or_path      ,      from_tf=bool(".ckpt" in model_args.model_name_or_path     )      ,      config=SCREAMING_SNAKE_CASE_      ,      cache_dir=model_args.cache_dir      ,      )

					# Get datasets
					__lowerCAmelCase				   =       (
					    MultipleChoiceDataset(
					        data_dir=data_args.data_dir      ,      tokenizer=SCREAMING_SNAKE_CASE_      ,      task=data_args.task_name      ,      max_seq_length=data_args.max_seq_length      ,      overwrite_cache=data_args.overwrite_cache      ,      mode=Split.train      ,      )
					    if training_args.do_train
					    else None
					)
					__lowerCAmelCase				   =       (
					    MultipleChoiceDataset(
					        data_dir=data_args.data_dir      ,      tokenizer=SCREAMING_SNAKE_CASE_      ,      task=data_args.task_name      ,      max_seq_length=data_args.max_seq_length      ,      overwrite_cache=data_args.overwrite_cache      ,      mode=Split.dev      ,      )
					    if training_args.do_eval
					    else None
					)

					def compute_metrics(SCREAMING_SNAKE_CASE_							:				EvalPrediction     ) -> Dict:
										__lowerCAmelCase				   =       np.argmax(p.predictions      ,      axis=1     )
										return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE_      ,      p.label_ids     )}

					# Data collator
					__lowerCAmelCase				   =       DataCollatorWithPadding(SCREAMING_SNAKE_CASE_      ,      pad_to_multiple_of=8     ) if training_args.fpaa else None

					# Initialize our Trainer
					__lowerCAmelCase				   =       Trainer(
					    model=SCREAMING_SNAKE_CASE_      ,      args=SCREAMING_SNAKE_CASE_      ,      train_dataset=SCREAMING_SNAKE_CASE_      ,      eval_dataset=SCREAMING_SNAKE_CASE_      ,      compute_metrics=SCREAMING_SNAKE_CASE_      ,      data_collator=SCREAMING_SNAKE_CASE_      ,      )

					# Training
					if training_args.do_train:
										trainer.train(
										    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
										trainer.save_model()
										# For convenience, we also re-save the tokenizer to the same directory,
										# so that you can share your model easily on huggingface.co/models =)
										if trainer.is_world_master():
															tokenizer.save_pretrained(training_args.output_dir     )

    # Evaluation
					__lowerCAmelCase				   =       {}
					if training_args.do_eval:
										logger.info("*** Evaluate ***"     )

										__lowerCAmelCase				   =       trainer.evaluate()

										__lowerCAmelCase				   =       os.path.join(training_args.output_dir      ,      "eval_results.txt"     )
										if trainer.is_world_master():
															with open(SCREAMING_SNAKE_CASE_      ,      "w"     ) as writer:
																				logger.info("***** Eval results *****"     )
																				for key, value in result.items():
																									logger.info("  %s = %s"      ,      SCREAMING_SNAKE_CASE_      ,      SCREAMING_SNAKE_CASE_     )
																									writer.write("%s = %s\n" % (key, value)     )

																				results.update(SCREAMING_SNAKE_CASE_     )

					return results







def 	_a      (						SCREAMING_SNAKE_CASE_							:				Optional[Any]     ):
					# For xla_spawn (TPUs)
					main()


if __name__ == "__main__":
					main()










import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    			=			get_tests_dir("""fixtures/test_sentencepiece.model""")
UpperCamelCase__    			=			get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

UpperCamelCase__    			=			"""pt""" if is_torch_available() else """tf"""
@require_sentencepiece
@require_tokenizers
class 			a__    (					snake_case__ ,				unittest.TestCase      ):
						_a							:    int					=    CamembertTokenizer
						_a							:    Dict					=    CamembertTokenizerFast
						_a							:    Tuple					=    True
						_a							:    List[Any]					=    True






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											__lowerCAmelCase				   =       CamembertTokenizer(_A			)
											tokenizer.save_pretrained(self.tmpdirname			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       "<pad>"
											__lowerCAmelCase				   =       1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A			)		,			_A			)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A			)		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       list(self.get_tokenizer().get_vocab().keys()			)

											self.assertEqual(vocab_keys[0]		,			"<s>NOTUSED"			)
											self.assertEqual(vocab_keys[1]		,			"<pad>"			)
											self.assertEqual(vocab_keys[-1]		,			"<mask>"			)
											self.assertEqual(len(_A			)		,			1_0_0_4			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size		,			1_0_0_5			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       CamembertTokenizer(_A			)
											tokenizer.save_pretrained(self.tmpdirname			)
											__lowerCAmelCase				   =       CamembertTokenizerFast.from_pretrained(self.tmpdirname			)

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											# <unk> tokens are not the same for `rust` than for `slow`.
											# Because spm gives back raw token instead of `unk` in EncodeAsPieces
											# tokens = tokenizer.tokenize(sequence)
											__lowerCAmelCase				   =       tokenizer.convert_ids_to_tokens(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)






						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											if not self.test_rust_tokenizer:
																return

											__lowerCAmelCase				   =       self.get_tokenizer()
											__lowerCAmelCase				   =       self.get_rust_tokenizer()

											__lowerCAmelCase				   =       "I was born in 92000, and this is falsé."

											__lowerCAmelCase				   =       tokenizer.tokenize(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.tokenize(_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       tokenizer.encode(_A		,			add_special_tokens=_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A		,			add_special_tokens=_A			)
											self.assertListEqual(_A		,			_A			)

											__lowerCAmelCase				   =       self.get_rust_tokenizer()
											__lowerCAmelCase				   =       tokenizer.encode(_A			)
											__lowerCAmelCase				   =       rust_tokenizer.encode(_A			)
											self.assertListEqual(_A		,			_A			)






						@slow
						def 				__SCREAMING_SNAKE_CASE(   self			):




											"""simple docstring"""



											__lowerCAmelCase				   =       {"input_ids": [[5, 5_4, 7_1_9_6, 2_9_7, 3_0, 2_3, 7_7_6, 1_8, 1_1, 3_2_1_5, 3_7_0_5, 8_2_5_2, 2_2, 3_1_6_4, 1_1_8_1, 2_1_1_6, 2_9, 1_6, 8_1_3, 2_5, 7_9_1, 3_3_1_4, 2_0, 3_4_4_6, 3_8, 2_7_5_7_5, 1_2_0, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_6_8, 1_7, 1_1, 9_0_8_8, 2_0, 1_5_1_7, 8, 2_2_8_0_4, 1_8_8_1_8, 1_0, 3_8, 6_2_9, 6_0_7, 6_0_7, 1_4_2, 1_9, 7_1_9_6, 8_6_7, 5_6, 1_0_3_2_6, 2_4, 2_2_6_7, 2_0, 4_1_6, 5_0_7_2, 1_5_6_1_2, 2_3_3, 7_3_4, 7, 2_3_9_9, 2_7, 1_6, 3_0_1_5, 1_6_4_9, 7, 2_4, 2_0, 4_3_3_8, 2_3_9_9, 2_7, 1_3, 3_4_0_0, 1_4, 1_3, 6_1_8_9, 8, 9_3_0, 9, 6]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
											# fmt: on

											# camembert is a french model. So we also use french texts.
											__lowerCAmelCase				   =       [
											    "Le transformeur est un modèle d'apprentissage profond introduit en 2017, "
											    "utilisé principalement dans le domaine du traitement automatique des langues (TAL).",
											    "À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus "
											    "pour gérer des données séquentielles, telles que le langage naturel, pour des tâches "
											    "telles que la traduction et la synthèse de texte.",
											]

											self.tokenizer_integration_test_util(
											    expected_encoding=_A		,			model_name="camembert-base"		,			revision="3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"		,			sequences=_A		,			)




