





import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from seqaseq_trainer import SeqaSeqTrainer
from seqaseq_training_args import SeqaSeqTrainingArguments

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSeqaSeqLM,
    AutoTokenizer,
    HfArgumentParser,
    MBartTokenizer,
    MBartTokenizerFast,
    set_seed,
)
from transformers.trainer_utils import EvaluationStrategy, is_main_process
from transformers.training_args import ParallelMode
from utils import (
    SeqaSeqDataCollator,
    SeqaSeqDataset,
    assert_all_frozen,
    build_compute_metrics_fn,
    check_output_dir,
    freeze_embeds,
    freeze_params,
    lmap,
    save_json,
    use_task_specific_params,
    write_txt_file,
)


snake_case_				=			logging.getLogger(__name__)

@dataclass
class 		SCREAMING_SNAKE_CASE__				:
      A_						:				str							=    field(
          metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}  )
      A_						:				Optional[str]							=    field(
          default=_UpperCAmelCase						,      metadata={'help': 'Pretrained config name or path if not the same as model_name'}  )
      A_						:				Optional[str]							=    field(
          default=_UpperCAmelCase						,      metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}  )
      A_						:				Optional[str]							=    field(
          default=_UpperCAmelCase						,      metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}						,      )
      A_						:				bool							=    field(default=_UpperCAmelCase						,      metadata={'help': 'Whether tp freeze the encoder.'}  )
      A_						:				bool							=    field(default=_UpperCAmelCase						,      metadata={'help': 'Whether  to freeze the embeddings.'}  )






@dataclass
class 		SCREAMING_SNAKE_CASE__				:
      A_						:				str							=    field(
          metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}  )
      A_						:				Optional[str]							=    field(
          default='summarization'						,      metadata={'help': 'Task name, summarization (or summarization_{dataset} for pegasus) or translation'}						,      )
      A_						:				Optional[int]							=    field(
          default=1_024						,      metadata={
              'help': (
                  'The maximum total input sequence length after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          }						,      )
      A_						:				Optional[int]							=    field(
          default=128						,      metadata={
              'help': (
                  'The maximum total sequence length for target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          }						,      )
      A_						:				Optional[int]							=    field(
          default=142						,      metadata={
              'help': (
                  'The maximum total sequence length for validation target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded. '
                  'This argument is also used to override the ``max_length`` param of ``model.generate``, which is used '
                  'during ``evaluate`` and ``predict``.'
              )
          }						,      )
      A_						:				Optional[int]							=    field(
          default=142						,      metadata={
              'help': (
                  'The maximum total sequence length for test target text after tokenization. Sequences longer '
                  'than this will be truncated, sequences shorter will be padded.'
              )
          }						,      )
      A_						:				Optional[int]							=    field(default=-1						,      metadata={'help': '# training examples. -1 means use all.'}  )
      A_						:				Optional[int]							=    field(default=-1						,      metadata={'help': '# validation examples. -1 means use all.'}  )
      A_						:				Optional[int]							=    field(default=-1						,      metadata={'help': '# test examples. -1 means use all.'}  )
      A_						:				Optional[str]							=    field(default=_UpperCAmelCase						,      metadata={'help': 'Source language id for translation.'}  )
      A_						:				Optional[str]							=    field(default=_UpperCAmelCase						,      metadata={'help': 'Target language id for translation.'}  )
      A_						:				Optional[int]							=    field(default=_UpperCAmelCase						,      metadata={'help': '# num_beams to use for evaluation.'}  )
      A_						:				bool							=    field(
          default=_UpperCAmelCase						,      metadata={'help': 'If only pad tokens should be ignored. This assumes that `config.pad_token_id` is defined.'}						,      )






def lowerCamelCase__    (							snake_case_	:  List[Any]       ,      snake_case_	:  List[str]       ,      snake_case_	:  Dict       )      ->				str:
   logger.info(f"""***** {split} metrics *****"""       )
   for key in sorted(metrics.keys()       ):
      logger.info(f"""  {key} = {metrics[key]}"""       )
   save_json(snake_case_       ,      os.path.join(snake_case_       ,      f"""{split}_results.json"""       )       )






def lowerCamelCase__    (							)      ->				Optional[Any]:
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   __snake_case							       =   HfArgumentParser((ModelArguments, DataTrainingArguments, SeqaSeqTrainingArguments)       )

   if len(sys.argv       ) == 2 and sys.argv[1].endswith('''.json'''       ):
      # If we pass only one argument to the script and it's the path to a json file,
      # let's parse it to get our arguments.
      __snake_case       ,	__snake_case       ,	__snake_case							       =   parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]       )       )
   else:
      __snake_case       ,	__snake_case       ,	__snake_case							       =   parser.parse_args_into_dataclasses()

   check_output_dir(snake_case_       )

   # Setup logging
   logging.basicConfig(
       format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''       ,      datefmt='''%m/%d/%Y %H:%M:%S'''       ,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,      )
   logger.warning(
       '''Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'''       ,      training_args.local_rank       ,      training_args.device       ,      training_args.n_gpu       ,      bool(training_args.parallel_mode == ParallelMode.DISTRIBUTED       )       ,      training_args.fpaa       ,      )
   transformers.utils.logging.enable_default_handler()
   transformers.utils.logging.enable_explicit_format()
   # Set the verbosity to info of the Transformers logger (on main process only):
   if is_main_process(training_args.local_rank       ):
      transformers.utils.logging.set_verbosity_info()
   logger.info('''Training/evaluation parameters %s'''       ,      snake_case_       )

   # Set seed
   set_seed(training_args.seed       )

   # Load pretrained model and tokenizer
   #
   # Distributed training:
   # The .from_pretrained methods guarantee that only one local process can concurrently
   # download model & vocab.

   __snake_case							       =   AutoConfig.from_pretrained(
       model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,      cache_dir=model_args.cache_dir       ,      )

   __snake_case							       =   ('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
   for p in extra_model_params:
      if getattr(snake_case_       ,      snake_case_       ,      snake_case_       ):
         assert hasattr(snake_case_       ,      snake_case_       ), f"""({config.__class__.__name__}) doesn't have a `{p}` attribute"""
         setattr(snake_case_       ,      snake_case_       ,      getattr(snake_case_       ,      snake_case_       )       )

   __snake_case							       =   AutoTokenizer.from_pretrained(
       model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,      cache_dir=model_args.cache_dir       ,      )
   __snake_case							       =   AutoModelForSeqaSeqLM.from_pretrained(
       model_args.model_name_or_path       ,      from_tf='''.ckpt''' in model_args.model_name_or_path       ,      config=snake_case_       ,      cache_dir=model_args.cache_dir       ,      )

   # use task specific params
   use_task_specific_params(snake_case_       ,      data_args.task       )

   # set num_beams for evaluation
   if data_args.eval_beams is None:
      __snake_case							       =   model.config.num_beams

   # set decoder_start_token_id for MBart
   if model.config.decoder_start_token_id is None and isinstance(snake_case_       ,      (MBartTokenizer, MBartTokenizerFast)       ):
      assert (
          data_args.tgt_lang is not None and data_args.src_lang is not None
      ), "mBart requires --tgt_lang and --src_lang"
      if isinstance(snake_case_       ,      snake_case_       ):
         __snake_case							       =   tokenizer.lang_code_to_id[data_args.tgt_lang]
      else:
         __snake_case							       =   tokenizer.convert_tokens_to_ids(data_args.tgt_lang       )

   if model_args.freeze_embeds:
      freeze_embeds(snake_case_       )
   if model_args.freeze_encoder:
      freeze_params(model.get_encoder()       )
      assert_all_frozen(model.get_encoder()       )

   __snake_case							       =   SeqaSeqDataset

   # Get datasets
   __snake_case							       =   (
       dataset_class(
           snake_case_       ,      type_path='''train'''       ,      data_dir=data_args.data_dir       ,      n_obs=data_args.n_train       ,      max_target_length=data_args.max_target_length       ,      max_source_length=data_args.max_source_length       ,      prefix=model.config.prefix or ''''''       ,      )
       if training_args.do_train
       else None
   )
   __snake_case							       =   (
       dataset_class(
           snake_case_       ,      type_path='''val'''       ,      data_dir=data_args.data_dir       ,      n_obs=data_args.n_val       ,      max_target_length=data_args.val_max_target_length       ,      max_source_length=data_args.max_source_length       ,      prefix=model.config.prefix or ''''''       ,      )
       if training_args.do_eval or training_args.evaluation_strategy != EvaluationStrategy.NO
       else None
   )
   __snake_case							       =   (
       dataset_class(
           snake_case_       ,      type_path='''test'''       ,      data_dir=data_args.data_dir       ,      n_obs=data_args.n_test       ,      max_target_length=data_args.test_max_target_length       ,      max_source_length=data_args.max_source_length       ,      prefix=model.config.prefix or ''''''       ,      )
       if training_args.do_predict
       else None
   )

   # Initialize our Trainer
   __snake_case							       =   (
       build_compute_metrics_fn(data_args.task       ,      snake_case_       ) if training_args.predict_with_generate else None
   )
   __snake_case							       =   SeqaSeqTrainer(
       model=snake_case_       ,      args=snake_case_       ,      data_args=snake_case_       ,      train_dataset=snake_case_       ,      eval_dataset=snake_case_       ,      data_collator=SeqaSeqDataCollator(
           snake_case_       ,      snake_case_       ,      model.config.decoder_start_token_id       ,      training_args.tpu_num_cores       )       ,      compute_metrics=snake_case_       ,      tokenizer=snake_case_       ,      )

   __snake_case							       =   {}
   # Training
   if training_args.do_train:
      logger.info('''*** Train ***'''       )

      __snake_case							       =   trainer.train(
          model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path       ) else None       )
      __snake_case							       =   train_result.metrics
      __snake_case							       =   data_args.n_train

      trainer.save_model()  # this also saves the tokenizer

      if trainer.is_world_process_zero():
         handle_metrics('''train'''       ,      snake_case_       ,      training_args.output_dir       )
         all_metrics.update(snake_case_       )

         # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
         trainer.state.save_to_json(os.path.join(training_args.output_dir       ,      '''trainer_state.json'''       )       )

         # For convenience, we also re-save the tokenizer to the same directory,
         # so that you can share your model easily on huggingface.co/models =)
         tokenizer.save_pretrained(training_args.output_dir       )

    # Evaluation
   if training_args.do_eval:
      logger.info('''*** Evaluate ***'''       )

      __snake_case							       =   trainer.evaluate(metric_key_prefix='''val'''       )
      __snake_case							       =   data_args.n_val
      __snake_case							       =   round(metrics['''val_loss''']       ,      4       )

      if trainer.is_world_process_zero():
         handle_metrics('''val'''       ,      snake_case_       ,      training_args.output_dir       )
         all_metrics.update(snake_case_       )

   if training_args.do_predict:
      logger.info('''*** Predict ***'''       )

      __snake_case							       =   trainer.predict(test_dataset=snake_case_       ,      metric_key_prefix='''test'''       )
      __snake_case							       =   test_output.metrics
      __snake_case							       =   data_args.n_test

      if trainer.is_world_process_zero():
         __snake_case							       =   round(metrics['''test_loss''']       ,      4       )
         handle_metrics('''test'''       ,      snake_case_       ,      training_args.output_dir       )
         all_metrics.update(snake_case_       )

         if training_args.predict_with_generate:
            __snake_case							       =   tokenizer.batch_decode(
                test_output.predictions       ,      skip_special_tokens=snake_case_       ,      clean_up_tokenization_spaces=snake_case_       )
            __snake_case							       =   lmap(str.strip       ,      snake_case_       )
            write_txt_file(snake_case_       ,      os.path.join(training_args.output_dir       ,      '''test_generations.txt'''       )       )

   if trainer.is_world_process_zero():
      save_json(snake_case_       ,      os.path.join(training_args.output_dir       ,      '''all_results.json'''       )       )

   return all_metrics






def lowerCamelCase__    (							snake_case_	:  Optional[Any]       )      ->				Tuple:
   # For xla_spawn (TPUs)
   main()


if __name__ == "__main__":
    main()








import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
					from PIL import Image

					from transformers import CLIPImageProcessor, CLIPProcessor


@require_vision
class   A			(unittest.TestCase    ):
			'''simple docstring'''




			def    a_       (			self   :    Union[str, Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   tempfile.mkdtemp()

					# fmt: off
					A__        =   ["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
					# fmt: on
					A__        =   dict(zip(__lowerCAmelCase							,	range(len(__lowerCAmelCase   )   )   )   )
					A__        =   ["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
					A__        =   {"""unk_token""": """<unk>"""}

					A__        =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["""vocab_file"""]   )
					A__        =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["""merges_file"""]   )
					with open(self.vocab_file							,	"""w"""							,	encoding="""utf-8"""   ) as fp:
							fp.write(json.dumps(__lowerCAmelCase   ) + """\n"""   )
					with open(self.merges_file							,	"""w"""							,	encoding="""utf-8"""   ) as fp:
							fp.write("""\n""".join(__lowerCAmelCase   )   )

					A__        =   {
					    """do_resize""": True,
					    """size""": 20,
					    """do_center_crop""": True,
					    """crop_size""": 18,
					    """do_normalize""": True,
					    """image_mean""": [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3],
					    """image_std""": [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1],
					}
					A__        =   os.path.join(self.tmpdirname							,	__lowerCAmelCase   )
					with open(self.image_processor_file							,	"""w"""							,	encoding="""utf-8"""   ) as fp:
							json.dump(__lowerCAmelCase							,	__lowerCAmelCase   )






			def    a_       (			self   :    Tuple							,	**__lowerCAmelCase   :    Dict   )	->	str:







					"""simple docstring"""






					return CLIPTokenizer.from_pretrained(self.tmpdirname							,	**__lowerCAmelCase   )






			def    a_       (			self   :    Union[str, Any]							,	**__lowerCAmelCase   :    Dict   )	->	List[str]:







					"""simple docstring"""






					return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,	**__lowerCAmelCase   )






			def    a_       (			self   :    List[str]							,	**__lowerCAmelCase   :    Optional[Any]   )	->	Dict:







					"""simple docstring"""






					return CLIPImageProcessor.from_pretrained(self.tmpdirname							,	**__lowerCAmelCase   )






			def    a_       (			self   :    str   )	->	Dict:







					"""simple docstring"""






					shutil.rmtree(self.tmpdirname   )






			def    a_       (			self   :    str   )	->	Any:







					"""simple docstring"""






					A__        =   [np.random.randint(2_55							,	size=(3, 30, 4_00)							,	dtype=np.uinta   )]

					A__        =   [Image.fromarray(np.moveaxis(__lowerCAmelCase							,	0							,	-1   )   ) for x in image_inputs]

					return image_inputs






			def    a_       (			self   :    Optional[int]   )	->	Tuple:







					"""simple docstring"""






					A__        =   self.get_tokenizer()
					A__        =   self.get_rust_tokenizer()
					A__        =   self.get_image_processor()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )
					processor_slow.save_pretrained(self.tmpdirname   )
					A__        =   CLIPProcessor.from_pretrained(self.tmpdirname							,	use_fast=__lowerCAmelCase   )

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )
					processor_fast.save_pretrained(self.tmpdirname   )
					A__        =   CLIPProcessor.from_pretrained(self.tmpdirname   )

					self.assertEqual(processor_slow.tokenizer.get_vocab()							,	tokenizer_slow.get_vocab()   )
					self.assertEqual(processor_fast.tokenizer.get_vocab()							,	tokenizer_fast.get_vocab()   )
					self.assertEqual(tokenizer_slow.get_vocab()							,	tokenizer_fast.get_vocab()   )
					self.assertIsInstance(processor_slow.tokenizer							,	__lowerCAmelCase   )
					self.assertIsInstance(processor_fast.tokenizer							,	__lowerCAmelCase   )

					self.assertEqual(processor_slow.image_processor.to_json_string()							,	image_processor.to_json_string()   )
					self.assertEqual(processor_fast.image_processor.to_json_string()							,	image_processor.to_json_string()   )
					self.assertIsInstance(processor_slow.image_processor							,	__lowerCAmelCase   )
					self.assertIsInstance(processor_fast.image_processor							,	__lowerCAmelCase   )






			def    a_       (			self   :    Optional[Any]   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   CLIPProcessor(tokenizer=self.get_tokenizer()							,	image_processor=self.get_image_processor()   )
					processor.save_pretrained(self.tmpdirname   )

					A__        =   self.get_tokenizer(bos_token="""(BOS)"""							,	eos_token="""(EOS)"""   )
					A__        =   self.get_image_processor(do_normalize=__lowerCAmelCase							,	padding_value=1.0   )

					A__        =   CLIPProcessor.from_pretrained(
					    self.tmpdirname							,	bos_token="""(BOS)"""							,	eos_token="""(EOS)"""							,	do_normalize=__lowerCAmelCase							,	padding_value=1.0   )

					self.assertEqual(processor.tokenizer.get_vocab()							,	tokenizer_add_kwargs.get_vocab()   )
					self.assertIsInstance(processor.tokenizer							,	__lowerCAmelCase   )

					self.assertEqual(processor.image_processor.to_json_string()							,	image_processor_add_kwargs.to_json_string()   )
					self.assertIsInstance(processor.image_processor							,	__lowerCAmelCase   )






			def    a_       (			self   :    List[Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   self.get_image_processor()
					A__        =   self.get_tokenizer()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )

					A__        =   self.prepare_image_inputs()

					A__        =   image_processor(__lowerCAmelCase							,	return_tensors="""np"""   )
					A__        =   processor(images=__lowerCAmelCase							,	return_tensors="""np"""   )

					for key in input_image_proc.keys():
							self.assertAlmostEqual(input_image_proc[key].sum()							,	input_processor[key].sum()							,	delta=1e-2   )






			def    a_       (			self   :    Optional[Any]   )	->	Any:







					"""simple docstring"""






					A__        =   self.get_image_processor()
					A__        =   self.get_tokenizer()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )

					A__        =   """lower newer"""

					A__        =   processor(text=__lowerCAmelCase   )

					A__        =   tokenizer(__lowerCAmelCase   )

					for key in encoded_tok.keys():
							self.assertListEqual(encoded_tok[key]							,	encoded_processor[key]   )






			def    a_       (			self   :    Union[str, Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   self.get_image_processor()
					A__        =   self.get_tokenizer()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )

					A__        =   """lower newer"""
					A__        =   self.prepare_image_inputs()

					A__        =   processor(text=__lowerCAmelCase							,	images=__lowerCAmelCase   )

					self.assertListEqual(list(inputs.keys()   )							,	["""input_ids""", """attention_mask""", """pixel_values"""]   )

					# test if it raises when no input is passed
					with pytest.raises(__lowerCAmelCase   ):
							processor()






			def    a_       (			self   :    Tuple   )	->	str:







					"""simple docstring"""






					A__        =   self.get_image_processor()
					A__        =   self.get_tokenizer()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )

					A__        =   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

					A__        =   processor.batch_decode(__lowerCAmelCase   )
					A__        =   tokenizer.batch_decode(__lowerCAmelCase   )

					self.assertListEqual(__lowerCAmelCase							,	__lowerCAmelCase   )







			def    a_       (			self   :    Optional[int]   )	->	str:







					"""simple docstring"""






					A__        =   self.get_image_processor()
					A__        =   self.get_tokenizer()

					A__        =   CLIPProcessor(tokenizer=__lowerCAmelCase							,	image_processor=__lowerCAmelCase   )

					A__        =   """lower newer"""
					A__        =   self.prepare_image_inputs()

					A__        =   processor(text=__lowerCAmelCase							,	images=__lowerCAmelCase   )

					self.assertListEqual(list(inputs.keys()   )							,	processor.model_input_names   )



