





"""simple docstring"""





import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      LiltForQuestionAnswering,
      LiltForSequenceClassification,
      LiltForTokenClassification,
      LiltModel,
  )
  from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST






class 					_lowerCAmelCase   :
       """simple docstring"""




       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase=1_3    , __UpperCAmelCase=7    , __UpperCAmelCase=True    , __UpperCAmelCase=True    , __UpperCAmelCase=True    , __UpperCAmelCase=True    , __UpperCAmelCase=9_9    , __UpperCAmelCase=2_4    , __UpperCAmelCase=2    , __UpperCAmelCase=6    , __UpperCAmelCase=3_7    , __UpperCAmelCase="gelu"    , __UpperCAmelCase=0.1    , __UpperCAmelCase=0.1    , __UpperCAmelCase=5_1_2    , __UpperCAmelCase=1_6    , __UpperCAmelCase=2    , __UpperCAmelCase=0.02    , __UpperCAmelCase=3    , __UpperCAmelCase=None    , __UpperCAmelCase=1_0_0_0    , ):



         '''simple docstring'''

         lowerCAmelCase__			:Optional[int]											=							parent
         lowerCAmelCase__			:Any											=							batch_size
         lowerCAmelCase__			:Union[str, Any]											=							seq_length
         lowerCAmelCase__			:Union[str, Any]											=							is_training
         lowerCAmelCase__			:Union[str, Any]											=							use_input_mask
         lowerCAmelCase__			:Dict											=							use_token_type_ids
         lowerCAmelCase__			:Optional[Any]											=							use_labels
         lowerCAmelCase__			:Tuple											=							vocab_size
         lowerCAmelCase__			:Any											=							hidden_size
         lowerCAmelCase__			:Dict											=							num_hidden_layers
         lowerCAmelCase__			:str											=							num_attention_heads
         lowerCAmelCase__			:Optional[int]											=							intermediate_size
         lowerCAmelCase__			:Optional[Any]											=							hidden_act
         lowerCAmelCase__			:Union[str, Any]											=							hidden_dropout_prob
         lowerCAmelCase__			:List[Any]											=							attention_probs_dropout_prob
         lowerCAmelCase__			:str											=							max_position_embeddings
         lowerCAmelCase__			:Optional[Any]											=							type_vocab_size
         lowerCAmelCase__			:Tuple											=							type_sequence_label_size
         lowerCAmelCase__			:Union[str, Any]											=							initializer_range
         lowerCAmelCase__			:str											=							num_labels
         lowerCAmelCase__			:int											=							scope
         lowerCAmelCase__			:str											=							range_bbox






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							ids_tensor([self.batch_size, self.seq_length]    , self.vocab_size      )

         lowerCAmelCase__			:Tuple											=							ids_tensor([self.batch_size, self.seq_length, 4]    , self.range_bbox      )
         # Ensure that bbox is legal
         for i in range(bbox.shape[0]      ):
           for j in range(bbox.shape[1]      ):
             if bbox[i, j, 3] < bbox[i, j, 1]:
               lowerCAmelCase__			:Union[str, Any]											=							bbox[i, j, 3]
               lowerCAmelCase__			:Optional[Any]											=							bbox[i, j, 1]
               lowerCAmelCase__			:Optional[Any]											=							t
             if bbox[i, j, 2] < bbox[i, j, 0]:
               lowerCAmelCase__			:Tuple											=							bbox[i, j, 2]
               lowerCAmelCase__			:Tuple											=							bbox[i, j, 0]
               lowerCAmelCase__			:Tuple											=							t

         lowerCAmelCase__			:str											=							None
         if self.use_input_mask:
           lowerCAmelCase__			:Any											=							ids_tensor([self.batch_size, self.seq_length]    , vocab_size=2      )

         lowerCAmelCase__			:Dict											=							None
         if self.use_token_type_ids:
           lowerCAmelCase__			:str											=							ids_tensor([self.batch_size, self.seq_length]    , self.type_vocab_size      )

         lowerCAmelCase__			:Any											=							None
         lowerCAmelCase__			:int											=							None
         if self.use_labels:
           lowerCAmelCase__			:Tuple											=							ids_tensor([self.batch_size]    , self.type_sequence_label_size      )
           lowerCAmelCase__			:Optional[int]											=							ids_tensor([self.batch_size, self.seq_length]    , self.num_labels      )

         lowerCAmelCase__			:List[Any]											=							self.get_config()

         return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels






       def      snake_case						(    self      ):



         '''simple docstring'''

         return LiltConfig(
             vocab_size=self.vocab_size    , hidden_size=self.hidden_size    , num_hidden_layers=self.num_hidden_layers    , num_attention_heads=self.num_attention_heads    , intermediate_size=self.intermediate_size    , hidden_act=self.hidden_act    , hidden_dropout_prob=self.hidden_dropout_prob    , attention_probs_dropout_prob=self.attention_probs_dropout_prob    , max_position_embeddings=self.max_position_embeddings    , type_vocab_size=self.type_vocab_size    , initializer_range=self.initializer_range    , )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							LiltModel(config=__UpperCAmelCase      )
         model.to(__UpperCAmelCase      )
         model.eval()
         lowerCAmelCase__			:List[str]											=							model(__UpperCAmelCase    , bbox=__UpperCAmelCase    , attention_mask=__UpperCAmelCase    , token_type_ids=__UpperCAmelCase      )
         lowerCAmelCase__			:Optional[int]											=							model(__UpperCAmelCase    , bbox=__UpperCAmelCase    , token_type_ids=__UpperCAmelCase      )
         lowerCAmelCase__			:int											=							model(__UpperCAmelCase    , bbox=__UpperCAmelCase      )

         self.parent.assertEqual(result.last_hidden_state.shape    , (self.batch_size, self.seq_length, self.hidden_size)      )
         self.parent.assertEqual(result.pooler_output.shape    , (self.batch_size, self.hidden_size)      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							self.num_labels
         lowerCAmelCase__			:Optional[int]											=							LiltForTokenClassification(config=__UpperCAmelCase      )
         model.to(__UpperCAmelCase      )
         model.eval()
         lowerCAmelCase__			:Optional[int]											=							model(
             __UpperCAmelCase    , bbox=__UpperCAmelCase    , attention_mask=__UpperCAmelCase    , token_type_ids=__UpperCAmelCase    , labels=__UpperCAmelCase      )
         self.parent.assertEqual(result.logits.shape    , (self.batch_size, self.seq_length, self.num_labels)      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							LiltForQuestionAnswering(config=__UpperCAmelCase      )
         model.to(__UpperCAmelCase      )
         model.eval()
         lowerCAmelCase__			:Optional[int]											=							model(
             __UpperCAmelCase    , bbox=__UpperCAmelCase    , attention_mask=__UpperCAmelCase    , token_type_ids=__UpperCAmelCase    , start_positions=__UpperCAmelCase    , end_positions=__UpperCAmelCase    , )
         self.parent.assertEqual(result.start_logits.shape    , (self.batch_size, self.seq_length)      )
         self.parent.assertEqual(result.end_logits.shape    , (self.batch_size, self.seq_length)      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							self.prepare_config_and_inputs()
         (
             (
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				(
             lowerCAmelCase__
         )							,				
         )			:Tuple											=							config_and_inputs
         lowerCAmelCase__			:Optional[Any]											=							{
             'input_ids': input_ids,
             'bbox': bbox,
             'token_type_ids': token_type_ids,
             'attention_mask': input_mask,
         }
         return config, inputs_dict






@require_torch
class 					_lowerCAmelCase   (     a  ,   a  ,   a  ,   unittest.TestCase       ):
       """simple docstring"""




       __magic_name__     :Optional[Any]		  =					(
           (
               LiltModel,
               LiltForSequenceClassification,
               LiltForTokenClassification,
               LiltForQuestionAnswering,
           )
           if is_torch_available()
           else ()
       )
       __magic_name__     :Union[str, Any]		  =					(
           {
               """feature-extraction""": LiltModel,
               """question-answering""": LiltForQuestionAnswering,
               """text-classification""": LiltForSequenceClassification,
               """token-classification""": LiltForTokenClassification,
               """zero-shot""": LiltForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       __magic_name__     :Tuple		  =					False
       __magic_name__     :str		  =					False






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase      ):



         '''simple docstring'''

         return True






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							LiltModelTester(self      )
         lowerCAmelCase__			:List[Any]											=							ConfigTester(self    , config_class=__UpperCAmelCase    , hidden_size=3_7      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         self.config_tester.run_common_tests()






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Optional[int]											=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Any											=							self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
           lowerCAmelCase__			:str											=							type
           self.model_tester.create_and_check_model(*__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Dict											=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase      )






       @slow
       def      snake_case						(    self      ):



         '''simple docstring'''

         for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           lowerCAmelCase__			:Optional[Any]											=							LiltModel.from_pretrained(__UpperCAmelCase      )
           self.assertIsNotNone(__UpperCAmelCase      )






@require_torch
@slow
class 					_lowerCAmelCase   (     unittest.TestCase       ):
       """simple docstring"""




       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							LiltModel.from_pretrained('SCUT-DLVCLab/lilt-roberta-en-base'      ).to(__UpperCAmelCase      )

         lowerCAmelCase__			:str											=							torch.tensor([[1, 2]]    , device=__UpperCAmelCase      )
         lowerCAmelCase__			:Any											=							torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]    , device=__UpperCAmelCase      )

         # forward pass
         with torch.no_grad():
           lowerCAmelCase__			:Dict											=							model(input_ids=__UpperCAmelCase    , bbox=__UpperCAmelCase      )

         lowerCAmelCase__			:Optional[int]											=							torch.Size([1, 2, 7_6_8]      )
         lowerCAmelCase__			:Optional[Any]											=							torch.tensor(
             [[-0.06_53, 0.09_50, -0.00_61], [-0.05_45, 0.09_26, -0.03_24]]    , device=__UpperCAmelCase    , )

         self.assertTrue(outputs.last_hidden_state.shape    , __UpperCAmelCase      )
         self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]    , __UpperCAmelCase    , atol=1E-3      )      )







"""simple docstring"""





from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml






class 					_lowerCAmelCase   (     yaml.SafeLoader       ):
       """simple docstring"""




       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							[self.constructed_objects[key_node] for key_node, _ in node.value]
         lowerCAmelCase__			:str											=							[tuple(__UpperCAmelCase      ) if isinstance(__UpperCAmelCase    , __UpperCAmelCase      ) else key for key in keys]
         lowerCAmelCase__			:Optional[int]											=							Counter(__UpperCAmelCase      )
         lowerCAmelCase__			:int											=							[key for key in counter if counter[key] > 1]
         if duplicate_keys:
           raise TypeError(F"Got duplicate yaml keys: {duplicate_keys}"      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase=False      ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							super().construct_mapping(__UpperCAmelCase    , deep=__UpperCAmelCase      )
         self._check_no_duplicates_on_constructed_node(__UpperCAmelCase      )
         return mapping






def  __A   (_SCREAMING_SNAKE_CASE			)					->Tuple[Optional[str], str]:





  """simple docstring"""


  lowerCAmelCase__			:Optional[Any]											=							list(readme_content.splitlines()			)
  if full_content and full_content[0] == "---" and "---" in full_content[1:]:
    lowerCAmelCase__			:Optional[int]											=							full_content[1:].index('---'			) + 1
    lowerCAmelCase__			:Union[str, Any]											=							'\n'.join(full_content[1:sep_idx]			)
    return yamlblock, "\n".join(full_content[sep_idx + 1 :]			)

  return None, "\n".join(_SCREAMING_SNAKE_CASE			)






class 					_lowerCAmelCase   (     a       ):
       """simple docstring"""




       __magic_name__     :List[str]		  =					{"""train_eval_index"""}  # train-eval-index in the YAML metadata






       @classmethod
       def      snake_case						(    cls    , __UpperCAmelCase      ):



         '''simple docstring'''

         with open(__UpperCAmelCase    , encoding='utf-8'      ) as readme_file:
           lowerCAmelCase__							,				lowerCAmelCase__			:Union[str, Any]											=							_split_yaml_from_readme(readme_file.read()      )
         if yaml_string is not None:
           return cls.from_yaml_string(__UpperCAmelCase      )
         else:
           return cls()






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         if path.exists():
           with open(__UpperCAmelCase    , encoding='utf-8'      ) as readme_file:
             lowerCAmelCase__			:Optional[Any]											=							readme_file.read()
         else:
           lowerCAmelCase__			:Union[str, Any]											=							None
         lowerCAmelCase__			:Union[str, Any]											=							self._to_readme(__UpperCAmelCase      )
         with open(__UpperCAmelCase    , 'w'    , encoding='utf-8'      ) as readme_file:
           readme_file.write(__UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         if readme_content is not None:
           lowerCAmelCase__							,				lowerCAmelCase__			:Optional[int]											=							_split_yaml_from_readme(__UpperCAmelCase      )
           lowerCAmelCase__			:Optional[Any]											=							'---\n' + self.to_yaml_string() + '---\n' + content
         else:
           lowerCAmelCase__			:str											=							'---\n' + self.to_yaml_string() + '---\n'
         return full_content






       @classmethod
       def      snake_case						(    cls    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:Dict											=							yaml.load(__UpperCAmelCase    , Loader=_NoDuplicateSafeLoader      ) or {}

         # Convert the YAML keys to DatasetMetadata fields
         lowerCAmelCase__			:int											=							{
             (key.replace('-'    , '_'      ) if key.replace('-'    , '_'      ) in cls._FIELDS_WITH_DASHES else key): value
             for key, value in metadata_dict.items()
         }
         return cls(**__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         return yaml.safe_dump(
             {
                 (key.replace('_'    , '-'      ) if key in self._FIELDS_WITH_DASHES else key): value
                 for key, value in self.items()
             }    , sort_keys=__UpperCAmelCase    , allow_unicode=__UpperCAmelCase    , encoding='utf-8'    , ).decode('utf-8'      )






__A											=		{
    """image-classification""": [],
    """translation""": [],
    """image-segmentation""": [],
    """fill-mask""": [],
    """automatic-speech-recognition""": [],
    """token-classification""": [],
    """sentence-similarity""": [],
    """audio-classification""": [],
    """question-answering""": [],
    """summarization""": [],
    """zero-shot-classification""": [],
    """table-to-text""": [],
    """feature-extraction""": [],
    """other""": [],
    """multiple-choice""": [],
    """text-classification""": [],
    """text-to-image""": [],
    """text2text-generation""": [],
    """zero-shot-image-classification""": [],
    """tabular-classification""": [],
    """tabular-regression""": [],
    """image-to-image""": [],
    """tabular-to-text""": [],
    """unconditional-image-generation""": [],
    """text-retrieval""": [],
    """text-to-speech""": [],
    """object-detection""": [],
    """audio-to-audio""": [],
    """text-generation""": [],
    """conversational""": [],
    """table-question-answering""": [],
    """visual-question-answering""": [],
    """image-to-text""": [],
    """reinforcement-learning""": [],
    """voice-activity-detection""": [],
    """time-series-forecasting""": [],
    """document-question-answering""": [],
}


if __name__ == "__main__":
  from argparse import ArgumentParser

  __A											=		ArgumentParser(usage="""Validate the yaml metadata block of a README.md file.""")
  ap.add_argument("""readme_filepath""")
  __A											=		ap.parse_args()

  __A											=		Path(args.readme_filepath)
  __A											=		DatasetMetadata.from_readme(readme_filepath)
  print(dataset_metadata)
  dataset_metadata.to_readme(readme_filepath)
