
"""simple docstring"""



import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     NezhaForMaskedLM,
     NezhaForMultipleChoice,
     NezhaForNextSentencePrediction,
     NezhaForPreTraining,
     NezhaForQuestionAnswering,
     NezhaForSequenceClassification,
     NezhaForTokenClassification,
     NezhaModel,
 )
 from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST

class       lowerCAmelCase__				:







     def __init__(							self			:			int    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			Optional[Any]=13    ,      _lowerCamelCase			:			int=7    ,      _lowerCamelCase			:			List[str]=True    ,      _lowerCamelCase			:			Union[str, Any]=True    ,      _lowerCamelCase			:			int=True    ,      _lowerCamelCase			:			List[Any]=True    ,      _lowerCamelCase			:			List[str]=99    ,      _lowerCamelCase			:			Tuple=32    ,      _lowerCamelCase			:			Optional[Any]=5    ,      _lowerCamelCase			:			Tuple=4    ,      _lowerCamelCase			:			Dict=37    ,      _lowerCamelCase			:			Optional[int]="gelu"    ,      _lowerCamelCase			:			int=0.1    ,      _lowerCamelCase			:			str=0.1    ,      _lowerCamelCase			:			List[Any]=128    ,      _lowerCamelCase			:			Any=32    ,      _lowerCamelCase			:			Any=16    ,      _lowerCamelCase			:			List[Any]=2    ,      _lowerCamelCase			:			Union[str, Any]=0.0_2    ,      _lowerCamelCase			:			Dict=3    ,      _lowerCamelCase			:			Tuple=4    ,      _lowerCamelCase			:			Tuple=None    ,      ):
       _snake_case					     =       parent
       _snake_case					     =       batch_size
       _snake_case					     =       seq_length
       _snake_case					     =       is_training
       _snake_case					     =       use_input_mask
       _snake_case					     =       use_token_type_ids
       _snake_case					     =       use_labels
       _snake_case					     =       vocab_size
       _snake_case					     =       hidden_size
       _snake_case					     =       num_hidden_layers
       _snake_case					     =       num_attention_heads
       _snake_case					     =       intermediate_size
       _snake_case					     =       hidden_act
       _snake_case					     =       hidden_dropout_prob
       _snake_case					     =       attention_probs_dropout_prob
       _snake_case					     =       max_position_embeddings
       _snake_case					     =       type_vocab_size
       _snake_case					     =       type_sequence_label_size
       _snake_case					     =       initializer_range
       _snake_case					     =       num_labels
       _snake_case					     =       num_choices
       _snake_case					     =       scope







     def      lowercase	(							self			:			List[str]			):
       _snake_case					     =       ids_tensor([self.batch_size, self.seq_length]    ,      self.vocab_size			)

       _snake_case					     =       None
       if self.use_input_mask:
         _snake_case					     =       random_attention_mask([self.batch_size, self.seq_length]			)

       _snake_case					     =       None
       if self.use_token_type_ids:
         _snake_case					     =       ids_tensor([self.batch_size, self.seq_length]    ,      self.type_vocab_size			)

       _snake_case					     =       None
       _snake_case					     =       None
       _snake_case					     =       None
       if self.use_labels:
         _snake_case					     =       ids_tensor([self.batch_size]    ,      self.type_sequence_label_size			)
         _snake_case					     =       ids_tensor([self.batch_size, self.seq_length]    ,      self.num_labels			)
         _snake_case					     =       ids_tensor([self.batch_size]    ,      self.num_choices			)

       _snake_case					     =       self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







     def      lowercase	(							self			:			Dict			):
       return NezhaConfig(
           vocab_size=self.vocab_size    ,      hidden_size=self.hidden_size    ,      num_hidden_layers=self.num_hidden_layers    ,      num_attention_heads=self.num_attention_heads    ,      intermediate_size=self.intermediate_size    ,      hidden_act=self.hidden_act    ,      hidden_dropout_prob=self.hidden_dropout_prob    ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,      max_position_embeddings=self.max_position_embeddings    ,      type_vocab_size=self.type_vocab_size    ,      is_decoder=_lowerCamelCase    ,      initializer_range=self.initializer_range    ,      )







     def      lowercase	(							self			:			Optional[int]			):
       (
           (
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					
       )					     =       self.prepare_config_and_inputs()

       _snake_case					     =       True
       _snake_case					     =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
       _snake_case					     =       ids_tensor([self.batch_size, self.seq_length]    ,      vocab_size=2			)

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )







     def      lowercase	(							self			:			str    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			List[str]			):
       _snake_case					     =       NezhaModel(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase			)
       _snake_case					     =       model(_lowerCamelCase    ,      token_type_ids=_lowerCamelCase			)
       _snake_case					     =       model(_lowerCamelCase			)
       self.parent.assertEqual(result.last_hidden_state.shape    ,      (self.batch_size, self.seq_length, self.hidden_size)			)
       self.parent.assertEqual(result.pooler_output.shape    ,      (self.batch_size, self.hidden_size)			)







     def      lowercase	(							self			:			Optional[Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			str    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			Optional[int]    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			Union[str, Any]    ,      ):
       _snake_case					     =       True
       _snake_case					     =       NezhaModel(_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      encoder_hidden_states=_lowerCamelCase    ,      encoder_attention_mask=_lowerCamelCase    ,      )
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      encoder_hidden_states=_lowerCamelCase    ,      )
       _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase			)
       self.parent.assertEqual(result.last_hidden_state.shape    ,      (self.batch_size, self.seq_length, self.hidden_size)			)
       self.parent.assertEqual(result.pooler_output.shape    ,      (self.batch_size, self.hidden_size)			)







     def      lowercase	(							self			:			int    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			str    ,      _lowerCamelCase			:			Optional[int]    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			Optional[int]			):
       _snake_case					     =       NezhaForMaskedLM(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase			)
       self.parent.assertEqual(result.logits.shape    ,      (self.batch_size, self.seq_length, self.vocab_size)			)







     def      lowercase	(							self			:			Optional[Any]    ,      _lowerCamelCase			:			Optional[int]    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			str    ,      _lowerCamelCase			:			int    ,      _lowerCamelCase			:			List[Any]			):
       _snake_case					     =       NezhaForNextSentencePrediction(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase    ,      )
       self.parent.assertEqual(result.logits.shape    ,      (self.batch_size, 2)			)







     def      lowercase	(							self			:			Union[str, Any]    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			str    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			Tuple			):
       _snake_case					     =       NezhaForPreTraining(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase    ,      next_sentence_label=_lowerCamelCase    ,      )
       self.parent.assertEqual(result.prediction_logits.shape    ,      (self.batch_size, self.seq_length, self.vocab_size)			)
       self.parent.assertEqual(result.seq_relationship_logits.shape    ,      (self.batch_size, 2)			)







     def      lowercase	(							self			:			Optional[Any]    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			Tuple    ,      _lowerCamelCase			:			int    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			Dict    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			int			):
       _snake_case					     =       NezhaForQuestionAnswering(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      start_positions=_lowerCamelCase    ,      end_positions=_lowerCamelCase    ,      )
       self.parent.assertEqual(result.start_logits.shape    ,      (self.batch_size, self.seq_length)			)
       self.parent.assertEqual(result.end_logits.shape    ,      (self.batch_size, self.seq_length)			)







     def      lowercase	(							self			:			List[str]    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			Tuple    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Optional[Any]			):
       _snake_case					     =       self.num_labels
       _snake_case					     =       NezhaForSequenceClassification(_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase			)
       self.parent.assertEqual(result.logits.shape    ,      (self.batch_size, self.num_labels)			)







     def      lowercase	(							self			:			int    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			List[Any]			):
       _snake_case					     =       self.num_labels
       _snake_case					     =       NezhaForTokenClassification(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase			)
       self.parent.assertEqual(result.logits.shape    ,      (self.batch_size, self.seq_length, self.num_labels)			)







     def      lowercase	(							self			:			str    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			Optional[Any]    ,      _lowerCamelCase			:			List[Any]    ,      _lowerCamelCase			:			int    ,      _lowerCamelCase			:			Union[str, Any]    ,      _lowerCamelCase			:			Tuple			):
       _snake_case					     =       self.num_choices
       _snake_case					     =       NezhaForMultipleChoice(config=_lowerCamelCase			)
       model.to(_lowerCamelCase			)
       model.eval()
       _snake_case					     =       input_ids.unsqueeze(1			).expand(-1    ,      self.num_choices    ,      -1			).contiguous()
       _snake_case					     =       token_type_ids.unsqueeze(1			).expand(-1    ,      self.num_choices    ,      -1			).contiguous()
       _snake_case					     =       input_mask.unsqueeze(1			).expand(-1    ,      self.num_choices    ,      -1			).contiguous()
       _snake_case					     =       model(
           _lowerCamelCase    ,      attention_mask=_lowerCamelCase    ,      token_type_ids=_lowerCamelCase    ,      labels=_lowerCamelCase    ,      )
       self.parent.assertEqual(result.logits.shape    ,      (self.batch_size, self.num_choices)			)







     def      lowercase	(							self			:			Union[str, Any]			):
       _snake_case					     =       self.prepare_config_and_inputs()
       (
           (
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					
       )					     =       config_and_inputs
       _snake_case					     =       {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict






@require_torch
class       lowerCAmelCase__				(						A_ ,	A_ ,	A_ ,	unittest.TestCase	):
     __a						=					(
         (
             NezhaModel,
             NezhaForMaskedLM,
             NezhaForMultipleChoice,
             NezhaForNextSentencePrediction,
             NezhaForPreTraining,
             NezhaForQuestionAnswering,
             NezhaForSequenceClassification,
             NezhaForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     __a						=					(
         {
             """feature-extraction""": NezhaModel,
             """fill-mask""": NezhaForMaskedLM,
             """question-answering""": NezhaForQuestionAnswering,
             """text-classification""": NezhaForSequenceClassification,
             """token-classification""": NezhaForTokenClassification,
             """zero-shot""": NezhaForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     __a						=					True







     def      lowercase	(							self			:			List[Any]    ,      _lowerCamelCase			:			Any    ,      _lowerCamelCase			:			List[str]    ,      _lowerCamelCase			:			Optional[Any]=False			):
       _snake_case					     =       super()._prepare_for_class(_lowerCamelCase    ,      _lowerCamelCase    ,      return_labels=_lowerCamelCase			)

       if return_labels:
         if model_class in get_values(_lowerCamelCase			):
           _snake_case					     =       torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)    ,      dtype=torch.long    ,      device=_lowerCamelCase			)
           _snake_case					     =       torch.zeros(
               self.model_tester.batch_size    ,      dtype=torch.long    ,      device=_lowerCamelCase			)
       return inputs_dict







     def      lowercase	(							self			:			Dict			):
       _snake_case					     =       NezhaModelTester(self			)
       _snake_case					     =       ConfigTester(self    ,      config_class=_lowerCamelCase    ,      hidden_size=37			)







     def      lowercase	(							self			:			Optional[int]			):
       self.config_tester.run_common_tests()







     def      lowercase	(							self			:			Union[str, Any]			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCamelCase			)







     def      lowercase	(							self			:			List[str]			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(*_lowerCamelCase			)







     def      lowercase	(							self			:			Tuple			):
       # This regression test was failing with PyTorch < 1.3
       (
           (
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					(
           _snake_case
       )     ,					
       )					     =       self.model_tester.prepare_config_and_inputs_for_decoder()

       _snake_case					     =       None

       self.model_tester.create_and_check_model_as_decoder(
           _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      _lowerCamelCase    ,      )







     def      lowercase	(							self			:			str			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase			)







     def      lowercase	(							self			:			Optional[int]			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase			)







     def      lowercase	(							self			:			Optional[int]			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_next_sequence_prediction(*_lowerCamelCase			)







     def      lowercase	(							self			:			Any			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*_lowerCamelCase			)







     def      lowercase	(							self			:			List[str]			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase			)







     def      lowercase	(							self			:			str			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase			)







     def      lowercase	(							self			:			int			):
       _snake_case					     =       self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase			)







     @slow
     def      lowercase	(							self			:			int			):
       for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _snake_case					     =       NezhaModel.from_pretrained(_lowerCamelCase			)
         self.assertIsNotNone(_lowerCamelCase			)







     @slow
     @require_torch_gpu
     def      lowercase	(							self			:			int			):
       _snake_case     ,					_snake_case					     =       self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
         # NezhaForMultipleChoice behaves incorrectly in JIT environments.
         if model_class == NezhaForMultipleChoice:
           return

         _snake_case					     =       True
         _snake_case					     =       model_class(config=_lowerCamelCase			)

         _snake_case					     =       self._prepare_for_class(_lowerCamelCase    ,      _lowerCamelCase			)
         _snake_case					     =       torch.jit.trace(
             _lowerCamelCase    ,      (inputs_dict['''input_ids'''].to('''cpu'''			), inputs_dict['''attention_mask'''].to('''cpu'''			))			)

         with tempfile.TemporaryDirectory() as tmp:
           torch.jit.save(_lowerCamelCase    ,      os.path.join(_lowerCamelCase    ,      '''bert.pt'''			)			)
           _snake_case					     =       torch.jit.load(os.path.join(_lowerCamelCase    ,      '''bert.pt'''			)    ,      map_location=_lowerCamelCase			)
           loaded(inputs_dict['''input_ids'''].to(_lowerCamelCase			)    ,      inputs_dict['''attention_mask'''].to(_lowerCamelCase			)			)






@require_torch
class       lowerCAmelCase__				(						unittest.TestCase	):







     @slow
     def      lowercase	(							self			:			List[str]			):
       _snake_case					     =       NezhaModel.from_pretrained('''sijunhe/nezha-cn-base'''			)
       _snake_case					     =       torch.tensor([[0, 1, 2, 3, 4, 5]]			)
       _snake_case					     =       torch.tensor([[0, 1, 1, 1, 1, 1]]			)
       with torch.no_grad():
         _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase			)[0]
       _snake_case					     =       torch.Size((1, 6, 768)			)
       self.assertEqual(output.shape    ,      _lowerCamelCase			)
       _snake_case					     =       torch.tensor([[[0.0_6_8_5, 0.2_4_4_1, 0.1_1_0_2], [0.0_6_0_0, 0.1_9_0_6, 0.1_3_4_9], [0.0_2_2_1, 0.0_8_1_9, 0.0_5_8_6]]]			)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,      _lowerCamelCase    ,      atol=1e-4			)			)







     @slow
     def      lowercase	(							self			:			Tuple			):
       _snake_case					     =       NezhaForMaskedLM.from_pretrained('''sijunhe/nezha-cn-base'''			)
       _snake_case					     =       torch.tensor([[0, 1, 2, 3, 4, 5]]			)
       _snake_case					     =       torch.tensor([[1, 1, 1, 1, 1, 1]]			)
       with torch.no_grad():
         _snake_case					     =       model(_lowerCamelCase    ,      attention_mask=_lowerCamelCase			)[0]
       _snake_case					     =       torch.Size((1, 6, 21128)			)
       self.assertEqual(output.shape    ,      _lowerCamelCase			)
       _snake_case					     =       torch.tensor(
           [[-2.7_9_3_9, -1.7_9_0_2, -2.2_1_8_9], [-2.8_5_8_5, -1.8_9_0_8, -2.3_7_2_3], [-2.6_4_9_9, -1.7_7_5_0, -2.2_5_5_8]]			)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,      _lowerCamelCase    ,      atol=1e-4			)			)





"""simple docstring"""



import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


UpperCAmelCase__          = 'http://www.mocksite.com/file1.txt'
UpperCAmelCase__          = '"text": ["foo", "foo"]'
UpperCAmelCase__          = '6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'

class       lowerCAmelCase__				:
     __a						=					200
     __a						=					{"""Content-Length""": """100"""}
     __a						=					{}







     def      lowercase	(							self			:			List[str]    ,      **_lowerCamelCase			:			List[str]			):
       return [bytes(_lowerCamelCase    ,      '''utf-8'''			)]






def 					_UpperCAmelCase      ( *__lowerCamelCase   :       List[str] ,					**__lowerCamelCase   :       Dict		)     ->					Dict:
  return MockResponse()
@pytest.mark.parametrize('''urls_type''' ,					[str, list, dict]		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Union[str, Any] ,					__lowerCamelCase   :       List[str] ,					__lowerCamelCase   :       str		)     ->					int:
  import requests

  monkeypatch.setattr(__lowerCamelCase ,					'''request''' ,					__lowerCamelCase		)

  _snake_case					     =       URL
  if issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       url
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       [url]
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       {'''train''': url}
  _snake_case					     =       '''dummy'''
  _snake_case					     =       '''downloads'''
  _snake_case					     =       tmp_path
  _snake_case					     =       DownloadConfig(
      cache_dir=os.path.join(__lowerCamelCase ,					__lowerCamelCase		) ,					use_etag=__lowerCamelCase ,					)
  _snake_case					     =       DownloadManager(dataset_name=__lowerCamelCase ,					download_config=__lowerCamelCase		)
  _snake_case					     =       dl_manager.download(__lowerCamelCase		)
  _snake_case					     =       urls
  for downloaded_paths in [downloaded_paths]:
    if isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      _snake_case					     =       [downloaded_paths]
      _snake_case					     =       [urls]
    elif isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      assert "train" in downloaded_paths.keys()
      _snake_case					     =       downloaded_paths.values()
      _snake_case					     =       urls.values()
    assert downloaded_paths
    for downloaded_path, input_url in zip(__lowerCamelCase ,					__lowerCamelCase		):
      assert downloaded_path == dl_manager.downloaded_paths[input_url]
      _snake_case					     =       Path(__lowerCamelCase		)
      _snake_case					     =       downloaded_path.parts
      assert parts[-1] == HASH
      assert parts[-2] == cache_subdir
      assert downloaded_path.exists()
      _snake_case					     =       downloaded_path.read_text()
      assert content == CONTENT
      _snake_case					     =       downloaded_path.with_suffix('''.json'''		)
      assert metadata_downloaded_path.exists()
      _snake_case					     =       json.loads(metadata_downloaded_path.read_text()		)
      assert metadata_content == {"url": URL, "etag": None}
@pytest.mark.parametrize('''paths_type''' ,					[str, list, dict]		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       int ,					__lowerCamelCase   :       str ,					__lowerCamelCase   :       Optional[int]		)     ->					int:
  _snake_case					     =       str(__lowerCamelCase		)
  if issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       filename
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       [filename]
  elif issubclass(__lowerCamelCase ,					__lowerCamelCase		):
    _snake_case					     =       {'''train''': filename}
  _snake_case					     =       '''dummy'''
  _snake_case					     =       xz_file.parent
  _snake_case					     =       '''extracted'''
  _snake_case					     =       DownloadConfig(
      cache_dir=__lowerCamelCase ,					use_etag=__lowerCamelCase ,					)
  _snake_case					     =       DownloadManager(dataset_name=__lowerCamelCase ,					download_config=__lowerCamelCase		)
  _snake_case					     =       dl_manager.extract(__lowerCamelCase		)
  _snake_case					     =       paths
  for extracted_paths in [extracted_paths]:
    if isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      _snake_case					     =       [extracted_paths]
      _snake_case					     =       [paths]
    elif isinstance(__lowerCamelCase ,					__lowerCamelCase		):
      assert "train" in extracted_paths.keys()
      _snake_case					     =       extracted_paths.values()
      _snake_case					     =       paths.values()
    assert extracted_paths
    for extracted_path, input_path in zip(__lowerCamelCase ,					__lowerCamelCase		):
      assert extracted_path == dl_manager.extracted_paths[input_path]
      _snake_case					     =       Path(__lowerCamelCase		)
      _snake_case					     =       extracted_path.parts
      assert parts[-1] == hash_url_to_filename(__lowerCamelCase ,					etag=__lowerCamelCase		)
      assert parts[-2] == extracted_subdir
      assert extracted_path.exists()
      _snake_case					     =       extracted_path.read_text()
      _snake_case					     =       text_file.read_text()
      assert extracted_file_content == expected_file_content
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Tuple ,					__lowerCamelCase   :       List[Any]		)     ->					Dict:
  assert path.endswith('''.jsonl'''		)
  for num_items, line in enumerate(__lowerCamelCase ,					start=1		):
    _snake_case					     =       json.loads(line.decode('''utf-8'''		)		)
    assert item.keys() == {"col_1", "col_2", "col_3"}
  assert num_items == 4
@pytest.mark.parametrize('''archive_jsonl''' ,					['''tar_jsonl_path''', '''zip_jsonl_path''']		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       Dict ,					__lowerCamelCase   :       str		)     ->					Dict:
  _snake_case					     =       request.getfixturevalue(__lowerCamelCase		)
  _snake_case					     =       DownloadManager()
  for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
    _test_jsonl(__lowerCamelCase ,					__lowerCamelCase		)
  assert num_jsonl == 2
@pytest.mark.parametrize('''archive_nested_jsonl''' ,					['''tar_nested_jsonl_path''', '''zip_nested_jsonl_path''']		)
def 					_UpperCAmelCase      ( __lowerCamelCase   :       str ,					__lowerCamelCase   :       List[Any]		)     ->					Tuple:
  _snake_case					     =       request.getfixturevalue(__lowerCamelCase		)
  _snake_case					     =       DownloadManager()
  for num_tar, (path, file) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
    for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(__lowerCamelCase		) ,					start=1		):
      _test_jsonl(__lowerCamelCase ,					__lowerCamelCase		)
  assert num_tar == 1
  assert num_jsonl == 2

def 					_UpperCAmelCase      ( __lowerCamelCase   :       Tuple		)     ->					List[Any]:
  _snake_case					     =       DownloadManager()
  for num_file, file in enumerate(dl_manager.iter_files(__lowerCamelCase		) ,					start=1		):
    assert os.path.basename(__lowerCamelCase		) == ("test.txt" if num_file == 1 else "train.txt")
  assert num_file == 2



